Merge branch 'master' of github.com:conda/conda into separate-conda-p…

…ackage-metadata

conda/core/package_cache_data.py

@@ -5,7 +5,7 @@
 
 import codecs
 from collections import defaultdict
-from errno import EACCES, ENOENT, EPERM
+from errno import EACCES, ENOENT, EPERM, EROFS
 from logging import getLogger
 from os import listdir
 from os.path import basename, dirname, getsize, join
@@ -384,7 +384,7 @@ def _make_single_record(self, package_filename):
                 try:
                     write_as_json_to_file(repodata_record_path, repodata_record)
                 except (IOError, OSError) as e:
-                    if e.errno in (EACCES, EPERM) and isdir(dirname(repodata_record_path)):
+                    if e.errno in (EACCES, EPERM, EROFS) and isdir(dirname(repodata_record_path)):
                         raise NotWritableError(repodata_record_path, e.errno, caused_by=e)
                     else:
                         raise

conda/core/subdir_data.py

@@ -6,7 +6,7 @@
 import bz2
 from collections import defaultdict
 from contextlib import closing
-from errno import EACCES, ENODEV, EPERM
+from errno import EACCES, ENODEV, EPERM, EROFS
 from genericpath import getmtime, isfile
 import hashlib
 from io import open as io_open
@@ -30,9 +30,9 @@
 from ..common.url import join_url, maybe_unquote
 from ..core.package_cache_data import PackageCacheData
 from ..exceptions import (CondaDependencyError, CondaHTTPError, CondaUpgradeError,
-                          NotWritableError, UnavailableInvalidChannel)
+                          NotWritableError, UnavailableInvalidChannel, ProxyError)
 from ..gateways.connection import (ConnectionError, HTTPError, InsecureRequestWarning,
-                                   InvalidSchema, SSLError)
+                                   InvalidSchema, SSLError, RequestsProxyError)
 from ..gateways.connection.session import CondaSession
 from ..gateways.disk import mkdir_p, mkdir_p_sudo_safe
 from ..gateways.disk.delete import rm_rf
@@ -230,7 +230,7 @@ def _load(self):
                 with io_open(self.cache_path_json, 'w') as fh:
                     fh.write(raw_repodata_str or '{}')
             except (IOError, OSError) as e:
-                if e.errno in (EACCES, EPERM):
+                if e.errno in (EACCES, EPERM, EROFS):
                     raise NotWritableError(self.cache_path_json, e.errno, caused_by=e)
                 else:
                     raise
@@ -447,6 +447,9 @@ def fetch_repodata_remote_request(url, etag, mod_stamp):
             log.debug(stringify(resp, content_max_len=256))
         resp.raise_for_status()
 
+    except RequestsProxyError:
+        raise ProxyError()   # see #3962
+
     except InvalidSchema as e:
         if 'SOCKS' in text_type(e):
             message = dals("""

conda/exceptions.py

@@ -387,8 +387,13 @@ def __init__(self, message, **kwargs):
 
 class ProxyError(CondaError):
     def __init__(self, message):
-        msg = '%s' % message
-        super(ProxyError, self).__init__(msg)
+        message = dals("""
+        Conda cannot proceed due to an error in your proxy configuration.
+        Check for typos and other configuration errors in any '.netrc' file in your home directory,
+        any environment variables ending in '_PROXY', and any other system-wide proxy
+        configuration settings.
+        """)
+        super(ProxyError, self).__init__(message)
 
 
 class CondaIOError(CondaError, IOError):
@@ -610,7 +615,7 @@ class UnsatisfiableError(CondaError):
         unsatisfiable specifications.
     """
 
-    def __init__(self, bad_deps, chains=True):
+    def __init__(self, bad_deps, chains=True, strict=False):
         from .models.match_spec import MatchSpec
 
         # Remove any target values from the MatchSpecs, convert to strings
@@ -653,6 +658,10 @@ def __init__(self, bad_deps, chains=True):
 others, or with the existing package set:%s
 Use "conda search <package> --info" to see the dependencies for each package.'''
         msg = msg % dashlist(bad_deps)
+        if strict:
+            msg += ('\nNote that strict channel priority may have removed '
+                    'packages required for satisfiability.')
+
         super(UnsatisfiableError, self).__init__(msg)
 
 

conda/gateways/connection/__init__.py

@@ -22,7 +22,7 @@ def should_bypass_proxies_patched(should_bypass_proxies_func, url, no_proxy=None
     from requests.adapters import BaseAdapter, HTTPAdapter
     from requests.auth import AuthBase, _basic_auth_str
     from requests.cookies import extract_cookies_to_jar
-    from requests.exceptions import InvalidSchema, SSLError
+    from requests.exceptions import InvalidSchema, SSLError, ProxyError as RequestsProxyError
     from requests.hooks import dispatch_hook
     from requests.models import Response
     from requests.packages.urllib3.exceptions import InsecureRequestWarning
@@ -39,7 +39,8 @@ def should_bypass_proxies_patched(should_bypass_proxies_func, url, no_proxy=None
     from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter
     from pip._vendor.requests.auth import AuthBase, _basic_auth_str
     from pip._vendor.requests.cookies import extract_cookies_to_jar
-    from pip._vendor.requests.exceptions import InvalidSchema, SSLError
+    from pip._vendor.requests.exceptions import (InvalidSchema, SSLError,
+                                                 ProxyError as RequestsProxyError)
     from pip._vendor.requests.hooks import dispatch_hook
     from pip._vendor.requests.models import Response
     from pip._vendor.requests.packages.urllib3.exceptions import InsecureRequestWarning
@@ -69,3 +70,4 @@ def should_bypass_proxies_patched(should_bypass_proxies_func, url, no_proxy=None
 InvalidSchema = InvalidSchema
 SSLError = SSLError
 InsecureRequestWarning = InsecureRequestWarning
+RequestsProxyError = RequestsProxyError

conda/gateways/connection/download.py

@@ -9,7 +9,8 @@
 import tempfile
 import warnings
 
-from . import ConnectionError, HTTPError, InsecureRequestWarning, InvalidSchema, SSLError
+from . import (ConnectionError, HTTPError, InsecureRequestWarning, InvalidSchema,
+               SSLError, RequestsProxyError)
 from .session import CondaSession
 from ..disk.delete import rm_rf
 from ... import CondaError
@@ -18,8 +19,8 @@
 from ...base.context import context
 from ...common.compat import text_type
 from ...common.io import time_recorder
-from ...exceptions import (BasicClobberError, ChecksumMismatchError, CondaDependencyError,
-                           CondaHTTPError, maybe_raise)
+from ...exceptions import (BasicClobberError, CondaDependencyError, CondaHTTPError,
+                           ChecksumMismatchError, maybe_raise, ProxyError)
 
 log = getLogger(__name__)
 
@@ -114,6 +115,9 @@ def download(
                 log.debug("size mismatch for download: %s (%s != %s)", url, actual_size, size)
                 raise ChecksumMismatchError(url, target_full_path, "size", size, actual_size)
 
+    except RequestsProxyError:
+        raise ProxyError()  # see #3962
+
     except InvalidSchema as e:
         if 'SOCKS' in text_type(e):
             message = dals("""

conda/gateways/disk/__init__.py

@@ -94,4 +94,9 @@ def mkdir_p_sudo_safe(path):
     if not on_win:
         # set newly-created directory permissions to 02775
         # https://github.com/conda/conda/issues/6610#issuecomment-354478489
-        os.chmod(path, 0o2775)
+        try:
+            os.chmod(path, 0o2775)
+        except (OSError, IOError):
+            log.trace("Failed to set permissions to 2775 on %s (%d %d)",
+                      path, e.errno, errorcode[e.errno])
+            pass

conda/gateways/disk/create.py

@@ -4,7 +4,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import codecs
-from errno import EACCES, EPERM
+from errno import EACCES, EPERM, EROFS
 from io import open
 from logging import getLogger
 import os
@@ -391,7 +391,7 @@ def create_package_cache_directory(pkgs_dir):
         touch(join(pkgs_dir, PACKAGE_CACHE_MAGIC_FILE), mkdir=True, sudo_safe=sudo_safe)
         touch(join(pkgs_dir, 'urls'), sudo_safe=sudo_safe)
     except (IOError, OSError) as e:
-        if e.errno in (EACCES, EPERM):
+        if e.errno in (EACCES, EPERM, EROFS):
             log.trace("cannot create package cache directory '%s'", pkgs_dir)
             return False
         else:
@@ -411,7 +411,7 @@ def create_envs_directory(envs_dir):
         sudo_safe = expand(envs_dir).startswith(expand('~'))
         touch(join(envs_dir, envs_dir_magic_file), mkdir=True, sudo_safe=sudo_safe)
     except (IOError, OSError) as e:
-        if e.errno in (EACCES, EPERM):
+        if e.errno in (EACCES, EPERM, EROFS):
             log.trace("cannot create envs directory '%s'", envs_dir)
             return False
         else:

conda/gateways/disk/permissions.py

@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: BSD-3-Clause
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from errno import EACCES, ENOENT, EPERM
+from errno import EACCES, ENOENT, EPERM, EROFS
 from itertools import chain
 from logging import getLogger
 from os import X_OK, access, chmod, lstat, walk
@@ -34,7 +34,7 @@ def make_writable(path):
         if eno in (ENOENT,):
             log.debug("tried to make writable, but didn't exist: %s", path)
             raise
-        elif eno in (EACCES, EPERM):
+        elif eno in (EACCES, EPERM, EROFS):
             log.debug("tried make writable but failed: %s\n%r", path, e)
             return False
         else:

conda/history.py

@@ -5,7 +5,7 @@
 
 from ast import literal_eval
 import codecs
-from errno import EACCES, EPERM
+from errno import EACCES, EPERM, EROFS
 import logging
 from operator import itemgetter
 import os
@@ -114,7 +114,7 @@ def update(self):
             curr = set(prefix_rec.dist_str() for prefix_rec in pd.iter_records())
             self.write_changes(last, curr)
         except EnvironmentError as e:
-            if e.errno in (EACCES, EPERM):
+            if e.errno in (EACCES, EPERM, EROFS):
                 raise NotWritableError(self.path, e.errno)
             else:
                 raise

conda/resolve.py

@@ -318,48 +318,102 @@ def find_conflicts(self, specs):
             above) that all of the specs depend on *but in different ways*. We
             then identify the dependency chains that lead to those packages.
         """
+        # if only a single package matches the spec use the packages depends
+        # rather than the spec itself
+        if len(specs) == 1:
+            matches = self.find_matches(specs[0])
+            if len(matches) == 1:
+                specs = self.ms_depends(matches[0])
+
+        strict_channel_priority = context.channel_priority == ChannelPriority.STRICT
+
+        def find_matches_with_strict(ms):
+            matches = self.find_matches(ms)
+            if not strict_channel_priority:
+                return matches
+            sole_source_channel_name = self._get_strict_channel(ms.name)
+            return tuple(f for f in matches if f.channel.name == sole_source_channel_name)
+
         sdeps = {}
         # For each spec, assemble a dictionary of dependencies, with package
         # name as key, and all of the matching packages as values.
-        for ms in specs:
-            rec = sdeps.setdefault(ms, {})
-            slist = [ms]
+        for top_level_spec in specs:
+            # find all packages matching a top level specification
+            top_level_pkgs = find_matches_with_strict(top_level_spec)
+            top_level_sdeps = {top_level_spec.name: set(top_level_pkgs)}
+
+            # find all depends specs for in top level packages
+            # find the depends names for each top level packages
+            second_level_specs = set()
+            top_level_pkg_dep_names = []
+            for pkg in top_level_pkgs:
+                pkg_deps = self.ms_depends(pkg)
+                second_level_specs.update(pkg_deps)
+                top_level_pkg_dep_names.append([d.name for d in pkg_deps])
+
+            # find all second level packages and their specs
+            slist = []
+            for ms in second_level_specs:
+                deps = top_level_sdeps.setdefault(ms.name, set())
+                for fkey in find_matches_with_strict(ms):
+                    deps.add(fkey)
+                    slist.extend(
+                        ms2 for ms2 in self.ms_depends(fkey) if ms2.name != top_level_spec.name)
+
+            # dependency names which appear in all top level packages
+            # have been fully considered and not additions should be make to
+            # the package list for that name
+            locked_names = [top_level_spec.name]
+            for name in top_level_pkg_dep_names[0]:
+                if all(name in names for names in top_level_pkg_dep_names):
+                    locked_names.append(name)
+
+            # build out the rest of the dependency tree
             while slist:
                 ms2 = slist.pop()
-                deps = rec.setdefault(ms2.name, set())
-                for fkey in self.find_matches(ms2):
+                if ms2.name in locked_names:
+                    continue
+                deps = top_level_sdeps.setdefault(ms2.name, set())
+                for fkey in find_matches_with_strict(ms2):
                     if fkey not in deps:
                         deps.add(fkey)
-                        slist.extend(ms3 for ms3 in self.ms_depends(fkey) if ms3.name != ms.name)
+                        slist.extend(ms3 for ms3 in self.ms_depends(fkey)
+                                     if ms3.name != top_level_spec.name)
+            sdeps[top_level_spec] = top_level_sdeps
 
-        # Find the list of dependencies they have in common. And for each of
-        # *those*, find the individual packages that they all share. Those need
-        # to be removed as conflict candidates.
-        commkeys = set.intersection(*(set(s.keys()) for s in sdeps.values()))
-        commkeys = {k: set.intersection(*(v[k] for v in sdeps.values())) for k in commkeys}
-
-        # and find the dependency chains that lead to them.
+        # find deps with zero intersection between specs which include that dep
         bad_deps = []
-        for ms, sdep in iteritems(sdeps):
+        deps = set()
+        for sdep in sdeps.values():
+            deps.update(sdep.keys())
+        for dep in deps:
+            sdeps_with_dep = {k: v.get(dep) for k, v in sdeps.items() if dep in v.keys()}
+            if len(sdeps_with_dep) <= 1:
+                continue
+            intersection = set.intersection(*sdeps_with_dep.values())
+            if len(intersection) != 0:
+                continue
             filter = {}
-            for mn, v in sdep.items():
-                if mn != ms.name and mn in commkeys:
-                    # Mark this package's "unique" dependencies as invalid
-                    for fkey in v - commkeys[mn]:
-                        filter[fkey] = False
-            # Find the dependencies that lead to those invalid choices
-            ndeps = set(self.invalid_chains(ms, filter, False))
-            # This may produce some additional invalid chains that we
-            # don't care about. Select only those that terminate in our
-            # predetermined set of "common" keys.
-            ndeps = [nd for nd in ndeps if nd[-1].name in commkeys]
-            if ndeps:
+            for fkeys in sdeps_with_dep.values():
+                for fkey in fkeys:
+                    filter[fkey] = False
+            for spec in sdeps_with_dep.keys():
+                ndeps = set(self.invalid_chains(spec, filter, False))
+                ndeps = [nd for nd in ndeps if nd[-1].name == dep]
                 bad_deps.extend(ndeps)
-            else:
-                # This means the package *itself* was the common conflict.
-                bad_deps.append((ms,))
-
-        raise UnsatisfiableError(bad_deps)
+        if not bad_deps:
+            for spec in specs:
+                filter = {}
+                for name, valid_pkgs in sdeps[spec].items():
+                    if name == spec.name:
+                        continue
+                    for fkey in self.find_matches(MatchSpec(name)):
+                        filter[fkey] = fkey in valid_pkgs
+                bad_deps.extend(self.invalid_chains(spec, filter, False))
+        if not bad_deps:
+            # no conflicting nor missing packages found, return the bad specs
+            bad_deps = [(ms, ) for ms in specs]
+        raise UnsatisfiableError(bad_deps, strict=strict_channel_priority)
 
     def _get_strict_channel(self, package_name):
         try:

tests/gateways/disk/test_permissions.py

@@ -6,7 +6,7 @@
 
 import errno
 import pytest
-from errno import ENOENT, EACCES
+from errno import ENOENT, EACCES, EROFS, EPERM
 from shutil import rmtree
 from contextlib import contextmanager
 from tempfile import gettempdir
@@ -104,6 +104,15 @@ def test_make_writable_doesnt_exist():
 
 
 def test_make_writable_dir_EPERM():
+    import conda.gateways.disk.permissions
+    from conda.gateways.disk.permissions import make_writable
+    with patch.object(conda.gateways.disk.permissions, 'chmod') as chmod_mock:
+        chmod_mock.side_effect = IOError(EPERM, 'some message', 'foo')
+        with tempdir() as td:
+            assert not make_writable(td)
+
+
+def test_make_writable_dir_EACCES():
     import conda.gateways.disk.permissions
     from conda.gateways.disk.permissions import make_writable
     with patch.object(conda.gateways.disk.permissions, 'chmod') as chmod_mock:
@@ -112,6 +121,15 @@ def test_make_writable_dir_EPERM():
             assert not make_writable(td)
 
 
+def test_make_writable_dir_EROFS():
+    import conda.gateways.disk.permissions
+    from conda.gateways.disk.permissions import make_writable
+    with patch.object(conda.gateways.disk.permissions, 'chmod') as chmod_mock:
+        chmod_mock.side_effect = IOError(EROFS, 'some message', 'foo')
+        with tempdir() as td:
+            assert not make_writable(td)
+
+
 def test_recursive_make_writable():
     from conda.gateways.disk.permissions import recursive_make_writable
     with tempdir() as td: