Merge pull request conda#3410 from kalefranz/ent-clone-fix

fix conda#3267 channels clone ent issue

Makefile

@@ -2,8 +2,9 @@ clean:
 	find . -name \*.py[cod] -delete
 	find . -name __pycache__ -delete
 	rm -rf *.egg-info* .cache build
-	rm -f .coverage junit.xml tmpfile.rc conda/.version
+	rm -f .coverage junit.xml tmpfile.rc conda/.version tempfile.rc coverage.xml
 	rm -rf auxlib bin conda/progressbar
+	rm -rf conda-forge\:\: file\: https\: local\:\: r\:\:
 
 
 clean-all: clean

conda/base/context.py

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function
 
+from collections import defaultdict
+
 import os
 import sys
-from conda._vendor.auxlib.path import expand
 from itertools import chain
 from logging import getLogger
 from os.path import abspath, basename, dirname, expanduser, isdir, join
@@ -12,11 +13,13 @@
 from .constants import DEFAULT_CHANNELS, DEFAULT_CHANNEL_ALIAS, ROOT_ENV_NAME, SEARCH_PATH, conda
 from .._vendor.auxlib.compat import NoneType, string_types
 from .._vendor.auxlib.ish import dals
+from .._vendor.auxlib.path import expand
 from .._vendor.toolz.itertoolz import concatv
 from ..common.configuration import (Configuration, MapParameter, PrimitiveParameter,
                                     SequenceParameter)
-from ..common.url import urlparse
+from ..common.url import urlparse, path_to_url
 from ..exceptions import CondaEnvironmentNotFoundError, CondaValueError
+from ..common.compat import iteritems
 
 log = getLogger(__name__)
 
@@ -57,9 +60,12 @@ class Context(Configuration):
     _root_dir = PrimitiveParameter(sys.prefix, aliases=('root_dir',))
 
     # channels
-    channel_alias = PrimitiveParameter(DEFAULT_CHANNEL_ALIAS)
     channels = SequenceParameter(string_types, default=('defaults',))
+    channel_alias = PrimitiveParameter(DEFAULT_CHANNEL_ALIAS)
+    migrated_channel_aliases = SequenceParameter(string_types)  # TODO: also take a list of strings  # NOQA
     default_channels = SequenceParameter(string_types, DEFAULT_CHANNELS)
+    custom_channels = MapParameter(string_types)
+    migrated_custom_channels = MapParameter(string_types)  # TODO: also take a list of strings
 
     # command line
     always_copy = PrimitiveParameter(False, aliases=('copy',))
@@ -194,6 +200,49 @@ def binstar_hosts(self):
                 'anaconda.org',
                 'binstar.org')
 
+    def _build_channel_map(self):
+        from ..models.channel import Channel
+        channel_map = defaultdict(list)
+        inverted_channel_map = dict()
+
+        # local
+        local = Channel(path_to_url(self.local_build_root))
+        channel_map['local'].append(local)
+        inverted_channel_map[local] = 'local'
+
+        # defaults
+        if self.default_channels:
+            for url in self.default_channels:
+                c = Channel(url)
+                channel_map['defaults'].append(c)
+                inverted_channel_map[c] = 'defaults'
+
+        # custom channels
+        for channel_name, url in iteritems(self.custom_channels):
+            c = Channel(url)
+            channel_map[channel_name].append(c)
+            inverted_channel_map[c] = channel_name
+
+        # mapped custom channels (legacy channels)
+        for channel_name, url in iteritems(self.migrated_custom_channels):
+            c = Channel(url)
+            inverted_channel_map[c] = channel_name
+
+        self._cache['channel_map'] = channel_map
+        self._cache['inverted_channel_map'] = inverted_channel_map
+
+    @property
+    def channel_map(self):
+        if 'channel_map' not in self._cache:
+            self._build_channel_map()
+        return self._cache['channel_map']
+
+    @property
+    def inverted_channel_map(self):
+        if 'inverted_channel_map' not in self._cache:
+            self._build_channel_map()
+        return self._cache['inverted_channel_map']
+
 
 def conda_in_private_env():
     # conda is located in its own private environment named '_conda'

conda/install.py

@@ -626,7 +626,7 @@ def add_cached_package(pdir, url, overwrite=False, urlstxt=False):
         return
     if url:
         url = url
-    _, schannel = Channel(url).url_channel_wtf
+    schannel = Channel(url).canonical_name
     prefix = '' if schannel == 'defaults' else schannel + '::'
     xkey = xpkg or (xdir + '.tar.bz2')
     fname_table_[xkey] = fname_table_[path_to_url(xkey)] = prefix

conda/models/channel.py

@@ -1,14 +1,19 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from collections import OrderedDict
+import re
 from itertools import chain
 from logging import getLogger
 from os.path import join
 
+try:
+    from cytoolz.functoolz import excepts
+except ImportError:
+    from .._vendor.toolz.functoolz import excepts
+
 from ..base.constants import PLATFORM_DIRECTORIES, RECOGNIZED_URL_SCHEMES
 from ..base.context import context
-from ..common.compat import with_metaclass
+from ..common.compat import odict, with_metaclass
 from ..common.url import is_url, path_to_url, urlparse, urlunparse
 
 log = getLogger(__name__)
@@ -34,8 +39,8 @@ def __call__(cls, value):
             return value
         elif value in Channel._cache_:
             return Channel._cache_[value]
-        elif value in _SPECIAL_CHANNELS:
-            self = object.__new__(_SPECIAL_CHANNELS[value])
+        elif value is None:
+            self = object.__new__(NoneChannel)
         elif value.endswith('.tar.bz2'):
             self = object.__new__(UrlChannel)
         elif has_scheme(value):
@@ -50,40 +55,57 @@ def __call__(cls, value):
 @with_metaclass(ChannelType)
 class Channel(object):
     _cache_ = dict()
-    _local_url = path_to_url(context.local_build_root)
     _channel_alias_netloc = urlparse(context.channel_alias).netloc
+    _old_channel_alias_netloc = tuple(urlparse(ca).netloc
+                                      for ca in context.migrated_channel_aliases)
 
     @staticmethod
     def _reset_state():
         Channel._cache_ = dict()
-        Channel._local_url = path_to_url(context.local_build_root)
         Channel._channel_alias_netloc = urlparse(context.channel_alias).netloc
+        Channel._old_channel_alias_netloc = tuple(urlparse(ca).netloc
+                                                  for ca in context.migrated_channel_aliases)
 
     @property
     def base_url(self):
-        return urlunparse((self._scheme, self._netloc, self._path, None, None, None))
+        _path = excepts(AttributeError, lambda: self._path.lstrip('/'))()
+        if self._netloc in Channel._old_channel_alias_netloc:
+            ca = Channel(context.channel_alias)
+            return urlunparse((ca._scheme, ca._netloc, _path, None, None, None))
+        else:
+            return urlunparse((self._scheme, self._netloc, _path, None, None, None))
 
     def __eq__(self, other):
         return self._netloc == other._netloc and self._path == other._path
 
+    def __hash__(self):
+        return hash((self._netloc, self._path))
+
     @property
     def canonical_name(self):
-        if any(self == Channel(c) for c in context.default_channels):
-            return 'defaults'
-        elif self == Channel(self._local_url):
-            return 'local'
-        elif self._netloc == Channel(context.channel_alias)._netloc:
-            # TODO: strip token
-            return self._path.lstrip('/')
+        if self in context.inverted_channel_map:
+            return context.inverted_channel_map[self]
+        elif self._netloc == Channel._channel_alias_netloc:
+            return self._path.strip('/')
+        elif self._netloc in Channel._old_channel_alias_netloc:
+            return self._path.strip('/')
         else:
             return self.base_url
 
+    def _urls_helper(self):
+        return [join_url(self.base_url, context.subdir), join_url(self.base_url, 'noarch')]
+        # if self._platform is None:
+        #     return [join_url(self.base_url, context.subdir), join_url(self.base_url, 'noarch')]
+        # else:
+        #     return [join_url(self.base_url, self._platform)]
+
     @property
     def urls(self):
-        if self._platform is None:
-            return [join_url(self.base_url, context.subdir), join_url(self.base_url, 'noarch')]
+        if self.canonical_name in context.channel_map:
+            url_channels = context.channel_map[self.canonical_name]
+            return list(chain.from_iterable(c._urls_helper() for c in url_channels))
         else:
-            return [join_url(self.base_url, self._platform)]
+            return self._urls_helper()
 
     @property
     def url_channel_wtf(self):
@@ -108,6 +130,12 @@ def split_platform(value):
             return value, None
 
 
+TOKEN_RE = re.compile(r'(/t/[a-z0-9A-Z-]+)?(\S*)?')
+def split_token(value):  # NOQA
+    token, path = TOKEN_RE.match(value).groups()
+    return token, path or '/'
+
+
 class UrlChannel(Channel):
 
     def __init__(self, url):
@@ -121,40 +149,32 @@ def __init__(self, url):
         parsed = urlparse(url)
         self._scheme = parsed.scheme
         self._netloc = parsed.netloc
-        self._path, self._platform = split_platform(parsed.path)
+        _path, self._platform = split_platform(parsed.path)
+        self._token, self._path = split_token(_path)
+
+    def __repr__(self):
+        return "UrlChannel(%s)" % urlunparse(('', self._netloc, self._path.lstrip('/'),
+                                              None, None, None)).lstrip('/')
 
 
 class NamedChannel(Channel):
 
     def __init__(self, name):
         log.debug("making channel object for named channel: %s", name)
         self._raw_value = name
-        parsed = urlparse(context.channel_alias)
+        if name in context.custom_channels:
+            parsed = urlparse(context.custom_channels[name])
+        elif name.split('/')[0] in context.custom_channels:
+            parsed = urlparse(context.custom_channels[name.split('/')[0]])
+        else:
+            parsed = urlparse(context.channel_alias)
         self._scheme = parsed.scheme
         self._netloc = parsed.netloc
-        self._path = join(parsed.path, name)
+        self._path = join(parsed.path or '/', name)
         self._platform = None
 
-
-class DefaultChannel(NamedChannel):
-
-    @property
-    def canonical_name(self):
-        return "defaults"
-
-    @property
-    def urls(self):
-        return list(chain.from_iterable(Channel(c).urls for c in context.default_channels))
-
-
-class LocalChannel(UrlChannel):
-
-    def __init__(self, _):
-        super(LocalChannel, self).__init__(path_to_url(context.local_build_root))
-
-    @property
-    def canonical_name(self):
-        return "local"
+    def __repr__(self):
+        return "NamedChannel(%s)" % self._raw_value
 
 
 class NoneChannel(NamedChannel):
@@ -174,7 +194,7 @@ def urls(self):
 
 def prioritize_channels(channels):
     # ('https://conda.anaconda.org/conda-forge/osx-64/', ('conda-forge', 1))
-    result = OrderedDict()
+    result = odict()
     for q, chn in enumerate(channels):
         channel = Channel(chn)
         for url in channel.urls:
@@ -186,10 +206,3 @@ def prioritize_channels(channels):
 
 def offline_keep(url):
     return not context.offline or not is_url(url) or url.startswith('file:/')
-
-
-_SPECIAL_CHANNELS = {
-    'defaults': DefaultChannel,
-    'local': LocalChannel,
-    None: NoneChannel,
-}

conda/plan.py

@@ -53,7 +53,7 @@ def channel_str(rec):
         if rec.get('schannel'):
             return rec['schannel']
         if rec.get('url'):
-            return Channel(rec['url']).url_channel_wtf[1]  # <-- same thing as canonical_name
+            return Channel(rec['url']).canonical_name
         if rec.get('channel'):
             return Channel(rec['channel']).canonical_name
         return '<unknown>'

tests/base/test_context.py

@@ -1,8 +1,59 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
+
+from conda._vendor.auxlib.ish import dals
+from conda.base.context import context, reset_context
+from conda.common.compat import odict
+from conda.common.configuration import YamlRawParameter
+from conda.common.yaml import yaml_load
+from conda.models.channel import Channel
 from logging import getLogger
 from unittest import TestCase
 
-from conda.base.constants import DEFAULT_CHANNEL_ALIAS
 
 log = getLogger(__name__)
+
+platform = context.subdir
+
+
+class ContextTests(TestCase):
+
+    def setUp(self):
+        string = dals("""
+        custom_channels:
+          darwin: https://some.url.somewhere/stuff
+          chuck: http://another.url:8080/with/path
+        migrated_custom_channels:
+          darwin: s3://just/cant
+          chuck: file:///var/lib/repo/
+        migrated_channel_aliases:
+          - https://conda.anaconda.org
+        channel_alias: ftp://new.url:8082
+        """)
+        reset_context()
+        rd = odict(testdata=YamlRawParameter.make_raw_parameters('testdata', yaml_load(string)))
+        context._add_raw_data(rd)
+        Channel._reset_state()
+
+    def tearDown(self):
+        reset_context()
+
+    def test_migrated_custom_channels(self):
+        assert Channel('https://some.url.somewhere/stuff/noarch/a-mighty-fine.tar.bz2').canonical_name == 'darwin'
+        assert Channel('s3://just/cant/noarch/a-mighty-fine.tar.bz2').canonical_name == 'darwin'
+        assert Channel('s3://just/cant/noarch/a-mighty-fine.tar.bz2').urls == [
+            'https://some.url.somewhere/stuff/%s/' % platform,
+            'https://some.url.somewhere/stuff/noarch/']
+
+    def test_old_channel_alias(self):
+        cf_urls = ["ftp://new.url:8082/conda-forge/%s/" % platform, "ftp://new.url:8082/conda-forge/noarch/"]
+        assert Channel('conda-forge').urls == cf_urls
+
+        url = "https://conda.anaconda.org/conda-forge/osx-64/some-great-package.tar.bz2"
+        assert Channel(url).canonical_name == 'conda-forge'
+        assert Channel(url).base_url == 'ftp://new.url:8082/conda-forge'
+        assert Channel(url).urls == cf_urls
+        assert Channel("https://conda.anaconda.org/conda-forge/label/dev/linux-64/"
+                       "some-great-package.tar.bz2").urls == [
+            "ftp://new.url:8082/conda-forge/label/dev/%s/" % platform,
+            "ftp://new.url:8082/conda-forge/label/dev/noarch/"]