replace listdir and glob with scandir

Signed-off-by: Kale Franz <[email protected]>

conda.recipe/meta.yaml

@@ -44,6 +44,7 @@ requirements:
     - pyopenssl >=16.2.0
     - requests >=2.18.4,<3
     - ruamel_yaml >=0.11.14,<0.16
+    - scandir                      # [py<35]
     - setuptools >=31.0.1
   run_constrained:
     - conda-build >=3

conda/activate.py

@@ -6,27 +6,27 @@
 
 from collections import OrderedDict
 from errno import ENOENT
-from glob import glob
 from itertools import chain
-from logging import getLogger
+import json
 import os
 from os.path import abspath, basename, dirname, expanduser, expandvars, isdir, join, exists
 import re
 import sys
 from textwrap import dedent
-import json
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 # Since we have to have configuration context here, anything imported by
 #   conda.base.context is fair game, but nothing more.
 from . import CONDA_PACKAGE_ROOT, CondaError
 from ._vendor.toolz import concatv, drop
 from ._vendor.auxlib.compat import Utf8NamedTemporaryFile
+from .base.constants import PREFIX_STATE_FILE, PACKAGE_ENV_VARS_DIR, CONDA_ENV_VARS_UNSET_VAR
 from .base.context import ROOT_ENV_NAME, context, locate_prefix_by_name
 from .common.compat import FILESYSTEM_ENCODING, PY2, iteritems, on_win, string_types, text_type
 from .common.path import paths_equal
-from .base.constants import PREFIX_STATE_FILE, PACKAGE_ENV_VARS_DIR, CONDA_ENV_VARS_UNSET_VAR
-
-log = getLogger(__name__)
 
 
 class _Activator(object):
@@ -619,7 +619,10 @@ def index_of_path(paths, test_path):
                 while last_idx is None and prefix_dirs_idx > -1:
                     last_idx = index_of_path(path_list, prefix_dirs[prefix_dirs_idx])
                     if last_idx is None:
-                        log.info("Did not find path entry {}".format(prefix_dirs[prefix_dirs_idx]))
+                        print(
+                            "Did not find path entry {0}".format(prefix_dirs[prefix_dirs_idx]),
+                            file=sys.stderr
+                        )
                     prefix_dirs_idx = prefix_dirs_idx - 1
                 # this compensates for an extra Library/bin dir entry from the interpreter on
                 #     windows.  If that entry isn't being added, it should have no effect.
@@ -696,14 +699,27 @@ def _prompt_modifier(self, prefix, conda_default_env):
             return ""
 
     def _get_activate_scripts(self, prefix):
-        return self.path_conversion(sorted(glob(join(
-            prefix, 'etc', 'conda', 'activate.d', '*' + self.script_extension
-        ))))
+        _script_extension = self.script_extension
+        se_len = -len(_script_extension)
+        try:
+            paths = (entry.path for entry in scandir(join(prefix, 'etc', 'conda', 'activate.d')))
+        except EnvironmentError:
+            return ()
+        return self.path_conversion(sorted(
+            p for p in paths if p[se_len:] == _script_extension
+        ))
 
     def _get_deactivate_scripts(self, prefix):
-        return self.path_conversion(sorted(glob(join(
-            prefix, 'etc', 'conda', 'deactivate.d', '*' + self.script_extension
-        )), reverse=True))
+        _script_extension = self.script_extension
+        se_len = -len(_script_extension)
+        try:
+            paths = (entry.path for entry in scandir(join(prefix, 'etc', 'conda', 'deactivate.d')))
+        except EnvironmentError:
+            return ()
+        return self.path_conversion(sorted(
+            (p for p in paths if p[se_len:] == _script_extension),
+            reverse=True
+        ))
 
     def _get_environment_env_vars(self, prefix):
         env_vars_file = join(prefix, PREFIX_STATE_FILE)
@@ -712,8 +728,8 @@ def _get_environment_env_vars(self, prefix):
 
         # First get env vars from packages
         if exists(pkg_env_var_dir):
-            for pkg_env_var_file in sorted(os.listdir(pkg_env_var_dir)):
-                with open(join(pkg_env_var_dir, pkg_env_var_file), 'r') as f:
+            for pkg_env_var_path in sorted(entry.path for entry in scandir(pkg_env_var_dir)):
+                with open(pkg_env_var_path, 'r') as f:
                     env_vars.update(json.loads(f.read(), object_pairs_hook=OrderedDict))
 
         # Then get env vars from environment specification

conda/cli/install.py

@@ -6,6 +6,10 @@
 from logging import getLogger
 import os
 from os.path import abspath, basename, exists, isdir, isfile, join
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from . import common
 from .common import check_non_admin
@@ -42,7 +46,7 @@ def check_prefix(prefix, json=False):
     if name == ROOT_ENV_NAME:
         error = "'%s' is a reserved environment name" % name
     if exists(prefix):
-        if isdir(prefix) and 'conda-meta' not in os.listdir(prefix):
+        if isdir(prefix) and 'conda-meta' not in tuple(entry.name for entry in scandir(prefix)):
             return None
         error = "prefix already exists: %s" % prefix
 

conda/common/_os/linux.py

@@ -5,9 +5,12 @@
 
 from collections import OrderedDict
 from genericpath import exists
-from glob import glob
 from logging import getLogger
 import sys
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from ..._vendor.auxlib.decorators import memoize
 from ..compat import iteritems
@@ -58,8 +61,7 @@ def linux_get_libc_version():
     # version refers to that of uClibc. readlink() can help to try to
     # figure out a better name instead.
     if family == 'NPTL':  # pragma: no cover
-        clibs = glob('/lib/libc.so*')
-        for clib in clibs:
+        for clib in (entry.path for entry in scandir("/lib") if entry.name[:7] == "libc.so"):
             clib = readlink(clib)
             if exists(clib):
                 if clib.startswith('libuClibc'):

conda/common/configuration.py

@@ -22,12 +22,15 @@
 except ImportError:
     from collections import Mapping
 import copy
-from glob import glob
 from itertools import chain
 from logging import getLogger
 from os import environ, stat
 from os.path import basename, join, expandvars
 from stat import S_IFDIR, S_IFMT, S_IFREG
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from enum import Enum, EnumMeta
 
@@ -440,8 +443,10 @@ def _file_yaml_loader(fullpath):
         yield fullpath, YamlRawParameter.make_raw_parameters_from_file(fullpath)
 
     def _dir_yaml_loader(fullpath):
-        for filepath in sorted(concatv(glob(join(fullpath, "*.yml")),
-                                       glob(join(fullpath, "*.yaml")))):
+        for filepath in sorted(
+            p for p in (entry.path for entry in scandir(fullpath))
+            if p[-4:] == ".yml" or p[-5:] == ".yaml"
+        ):
             yield filepath, YamlRawParameter.make_raw_parameters_from_file(filepath)
 
     # map a stat result to a file loader or a directory loader

conda/common/pkg_formats/python.py

@@ -7,15 +7,18 @@
 from csv import reader as csv_reader
 from email.parser import HeaderParser
 from errno import ENOENT
-from fnmatch import filter as fnmatch_filter
 from logging import getLogger
-from os import listdir, name as os_name, strerror
+from os import name as os_name, strerror
 from os.path import basename, dirname, isdir, isfile, join, lexists
 import platform
 from posixpath import normpath as posix_normpath
 import re
 import sys
 import warnings
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from ... import CondaError
 from ..compat import PY2, StringIO, itervalues, odict, open, string_types
@@ -874,7 +877,8 @@ def parse_specification(spec):
 def get_site_packages_anchor_files(site_packages_path, site_packages_dir):
     """Get all the anchor files for the site packages directory."""
     site_packages_anchor_files = set()
-    for fname in listdir(site_packages_path):
+    for entry in scandir(site_packages_path):
+        fname = entry.name
         anchor_file = None
         if fname.endswith('.dist-info'):
             anchor_file = "%s/%s/%s" % (site_packages_dir, fname, 'RECORD')
@@ -921,7 +925,11 @@ def get_dist_file_from_egg_link(egg_link_file, prefix_path):
             egg_link_contents = fh.readlines()[0].strip()
 
     if lexists(egg_link_contents):
-        egg_info_fnames = fnmatch_filter(listdir(egg_link_contents), '*.egg-info')
+        egg_info_fnames = tuple(
+            name for name in
+            (entry.name for entry in scandir(egg_link_contents))
+            if name[-9:] == ".egg-info"
+        )
     else:
         egg_info_fnames = ()
 

conda/core/envs_manager.py

@@ -5,8 +5,12 @@
 
 from errno import EACCES, EROFS, ENOENT
 from logging import getLogger
-from os import devnull, listdir
+from os import devnull
 from os.path import dirname, isdir, isfile, join, normpath
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from .prefix_data import PrefixData
 from ..base.context import context
@@ -58,7 +62,7 @@ def unregister_env(location):
     if isdir(location):
         meta_dir = join(location, 'conda-meta')
         if isdir(meta_dir):
-            meta_dir_contents = listdir(meta_dir)
+            meta_dir_contents = tuple(entry.name for entry in scandir(meta_dir))
             if len(meta_dir_contents) > 1:
                 # if there are any files left other than 'conda-meta/history'
                 #   then don't unregister
@@ -73,7 +77,7 @@ def list_all_known_prefixes():
     if is_admin():
         if on_win:
             home_dir_dir = dirname(expand('~'))
-            search_dirs = tuple(join(home_dir_dir, d) for d in listdir(home_dir_dir))
+            search_dirs = tuple(entry.path for entry in scandir(home_dir_dir))
         else:
             from pwd import getpwall
             search_dirs = tuple(pwentry.pw_dir for pwentry in getpwall()) or (expand('~'),)
@@ -88,7 +92,7 @@ def list_all_known_prefixes():
     # all envs_dirs
     envs_dirs = (envs_dir for envs_dir in context.envs_dirs if isdir(envs_dir))
     all_env_paths.update(path for path in (
-        join(envs_dir, name) for envs_dir in envs_dirs for name in listdir(envs_dir)
+        entry.path for envs_dir in envs_dirs for entry in scandir(envs_dir)
     ) if path not in all_env_paths and is_conda_environment(path))
 
     all_env_paths.add(context.root_prefix)

conda/core/package_cache_data.py

@@ -7,10 +7,13 @@
 from collections import defaultdict
 from errno import EACCES, ENOENT, EPERM, EROFS
 from logging import getLogger
-from os import listdir
 from os.path import basename, dirname, getsize, join
 from sys import platform
 from tarfile import ReadError
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from .path_actions import CacheUrlAction, ExtractPackageAction
 from .. import CondaError, CondaMultiError, conda_signal_handler
@@ -87,7 +90,8 @@ def load(self):
             return
 
         _CONDA_TARBALL_EXTENSIONS = CONDA_PACKAGE_EXTENSIONS
-        for base_name in self._dedupe_pkgs_dir_contents(listdir(self.pkgs_dir)):
+        pkgs_dir_contents = tuple(entry.name for entry in scandir(self.pkgs_dir))
+        for base_name in self._dedupe_pkgs_dir_contents(pkgs_dir_contents):
             full_path = join(self.pkgs_dir, base_name)
             if islink(full_path):
                 continue

conda/core/prefix_data.py

@@ -3,13 +3,15 @@
 # SPDX-License-Identifier: BSD-3-Clause
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from fnmatch import filter as fnmatch_filter
+from collections import OrderedDict
+import json
 from logging import getLogger
-from os import listdir
 from os.path import basename, isdir, isfile, join, lexists
 import re
-from collections import OrderedDict
-import json
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from ..base.constants import PREFIX_STATE_FILE
 from .._vendor.auxlib.exceptions import ValidationError
@@ -69,8 +71,13 @@ def load(self):
         self.__prefix_records = {}
         _conda_meta_dir = join(self.prefix_path, 'conda-meta')
         if lexists(_conda_meta_dir):
-            for meta_file in fnmatch_filter(listdir(_conda_meta_dir), '*.json'):
-                self._load_single_record(join(_conda_meta_dir, meta_file))
+            conda_meta_json_paths = (
+                p for p in
+                (entry.path for entry in scandir(_conda_meta_dir))
+                if p[-5:] == ".json"
+            )
+            for meta_file in conda_meta_json_paths:
+                self._load_single_record(meta_file)
         if self._pip_interop_enabled:
             self._load_site_packages()
 

conda/gateways/anaconda_client.py

@@ -5,10 +5,13 @@
 
 import os
 import re
-from glob import glob
 from logging import getLogger
 from os.path import basename, isdir, isfile, join
 from stat import S_IREAD, S_IWRITE
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from .disk.delete import rm_rf
 from .._vendor.appdirs import AppDirs, EnvAppDirs
@@ -36,10 +39,11 @@ def read_binstar_tokens():
     if not isdir(token_dir):
         return tokens
 
-    token_files = glob(join(token_dir, '*.token'))
-    for tkn_file in token_files:
-        url = re.sub(r'\.token$', '', unquote_plus(basename(tkn_file)))
-        with open(tkn_file) as f:
+    for tkn_entry in scandir(token_dir):
+        if tkn_entry.name[-6:] != ".token":
+            continue
+        url = re.sub(r'\.token$', '', unquote_plus(tkn_entry.name))
+        with open(tkn_entry.path) as f:
             token = f.read()
         tokens[url] = tokens[replace_first_api_with_conda(url)] = token
     return tokens

conda/gateways/disk/delete.py

@@ -6,11 +6,15 @@
 from errno import ENOENT
 import fnmatch
 from logging import getLogger
-from os import environ, getcwd, listdir, makedirs, rename, rmdir, unlink, walk
+from os import environ, getcwd, makedirs, rename, rmdir, unlink, walk
 from os.path import abspath, basename, dirname, exists, isdir, isfile, join, normpath, split
 import shutil
 from subprocess import CalledProcessError, STDOUT, check_output
 import sys
+try:
+    from os import scandir
+except ImportError:
+    from scandir import scandir
 
 from . import MAX_TRIES, exp_backoff_fn
 from .link import islink, lexists
@@ -143,7 +147,8 @@ def unlink_or_rename_to_trash(path):
 def remove_empty_parent_paths(path):
     # recurse to clean up empty folders that were created to have a nested hierarchy
     parent_path = dirname(path)
-    while(isdir(parent_path) and not listdir(parent_path)):
+
+    while isdir(parent_path) and not next(scandir(parent_path), None):
         rmdir(parent_path)
         parent_path = dirname(parent_path)