Merge pull request conda#9835 from mrocklin/support-web-environments

resolve conda#5562 support creating conda environments from web addresses

conda/gateways/connection/download.py

@@ -186,6 +186,53 @@ def download(
                              caused_by=e)
 
 
+def download_text(url):
+    if sys.platform == 'win32':
+        preload_openssl()
+    if not context.ssl_verify:
+        disable_ssl_verify_warning()
+    try:
+        timeout = context.remote_connect_timeout_secs, context.remote_read_timeout_secs
+        session = CondaSession()
+        response = session.get(url, stream=True, proxies=session.proxies, timeout=timeout)
+        if log.isEnabledFor(DEBUG):
+            log.debug(stringify(response, content_max_len=256))
+        response.raise_for_status()
+    except RequestsProxyError:
+        raise ProxyError()  # see #3962
+    except InvalidSchema as e:
+        if 'SOCKS' in text_type(e):
+            message = dals("""
+                Requests has identified that your current working environment is configured
+                to use a SOCKS proxy, but pysocks is not installed.  To proceed, remove your
+                proxy configuration, run `conda install pysocks`, and then you can re-enable
+                your proxy configuration.
+                """)
+            raise CondaDependencyError(message)
+        else:
+            raise
+    except (ConnectionError, HTTPError, SSLError) as e:
+        status_code = getattr(e.response, 'status_code', None)
+        if status_code == 404:
+            help_message = dals("""
+            An HTTP error occurred when trying to retrieve this URL.
+            The URL does not exist.
+            """)
+        else:
+            help_message = dals("""
+            An HTTP error occurred when trying to retrieve this URL.
+            HTTP errors are often intermittent, and a simple retry will get you on your way.
+            """)
+        raise CondaHTTPError(help_message,
+                             url,
+                             status_code,
+                             getattr(e.response, 'reason', None),
+                             getattr(e.response, 'elapsed', None),
+                             e.response,
+                             caused_by=e)
+    return response.text
+
+
 class TmpDownload(object):
     """
     Context manager to handle downloads to a tempfile

conda/gateways/connection/session.py

@@ -24,6 +24,14 @@
 RETRIES = 3
 
 
+CONDA_SESSION_SCHEMES = frozenset((
+    "http",
+    "https",
+    "ftp",
+    "s3",
+    "file",
+))
+
 class EnforceUnusedAdapter(BaseAdapter):
 
     def send(self, request, *args, **kwargs):

conda_env/cli/main_create.py

@@ -11,6 +11,7 @@
 from conda._vendor.auxlib.path import expand
 from conda.cli import install as cli_install
 from conda.cli.conda_argparse import add_parser_json, add_parser_prefix, add_parser_networking
+from conda.gateways.connection.session import CONDA_SESSION_SCHEMES
 from conda.gateways.disk.delete import rm_rf
 from conda.misc import touch_nonadmin
 from .common import get_prefix, print_result
@@ -76,8 +77,13 @@ def execute(args, parser):
     name = args.remote_definition or args.name
 
     try:
-        spec = specs.detect(name=name, filename=expand(args.file),
-                            directory=os.getcwd())
+        url_scheme = args.file.split("://", 1)[0]
+        if url_scheme in CONDA_SESSION_SCHEMES:
+            filename = args.file
+        else:
+            filename = expand(args.file)
+
+        spec = specs.detect(name=name, filename=filename, directory=os.getcwd())
         env = spec.environment
 
         # FIXME conda code currently requires args to have a name or prefix

conda_env/env.py

@@ -13,6 +13,8 @@
 from conda.cli import common  # TODO: this should never have to import form conda.cli
 from conda.common.serialize import yaml_load_standard
 from conda.core.prefix_data import PrefixData
+from conda.gateways.connection.download import download_text
+from conda.gateways.connection.session import CONDA_SESSION_SCHEMES
 from conda.models.enums import PackageType
 from conda.models.match_spec import MatchSpec
 from conda.models.prefix_graph import PrefixGraph
@@ -144,11 +146,15 @@ def from_yaml(yamlstr, **kwargs):
 
 
 def from_file(filename):
-    if not os.path.exists(filename):
+    url_scheme = filename.split("://", 1)[0]
+    if url_scheme in CONDA_SESSION_SCHEMES:
+        yamlstr = download_text(filename)
+    elif not os.path.exists(filename):
         raise exceptions.EnvironmentFileNotFound(filename)
-    with open(filename, 'r') as fp:
-        yamlstr = fp.read()
-        return from_yaml(yamlstr, filename=filename)
+    else:
+        with open(filename, 'r') as fp:
+            yamlstr = fp.read()
+    return from_yaml(yamlstr, filename=filename)
 
 
 # TODO test explicitly

conda_env/installers/pip.py

@@ -6,6 +6,7 @@
 import os
 import os.path as op
 from conda._vendor.auxlib.compat import Utf8NamedTemporaryFile
+from conda.gateways.connection.session import CONDA_SESSION_SCHEMES
 from conda_env.pip_util import pip_subprocess, get_pip_installed_packages
 from logging import getLogger
 
@@ -27,10 +28,14 @@ def _pip_install_via_requirements(prefix, specs, args, *_, **kwargs):
       See: https://pip.pypa.io/en/stable/user_guide/#requirements-files
            https://pip.pypa.io/en/stable/reference/pip_install/#requirements-file-format
     """
-    try:
-        pip_workdir = op.dirname(op.abspath(args.file))
-    except AttributeError:
+    url_scheme = args.file.split("://", 1)[0]
+    if url_scheme in CONDA_SESSION_SCHEMES:
         pip_workdir = None
+    else:
+        try:
+            pip_workdir = op.dirname(op.abspath(args.file))
+        except AttributeError:
+            pip_workdir = None
     requirements = None
     try:
         # Generate the temporary requirements file

conda_env/specs/__init__.py

@@ -4,6 +4,7 @@
 
 import os
 
+from conda.gateways.connection.session import CONDA_SESSION_SCHEMES
 from .binstar import BinstarSpec
 from .notebook import NotebookSpec
 from .requirements import RequirementsSpec
@@ -13,18 +14,20 @@
 
 
 def detect(**kwargs):
-    filename = kwargs.get('filename')
+    filename = kwargs.get('filename', '')
     remote_definition = kwargs.get('name')
 
     # Check extensions
     all_valid_exts = YamlFileSpec.extensions.union(RequirementsSpec.extensions)
     fname, ext = os.path.splitext(filename)
 
     # First check if file exists and test the known valid extension for specs
-    file_exists = filename and os.path.isfile(filename)
+    file_exists = (
+        os.path.isfile(filename) or filename.split("://", 1)[0] in CONDA_SESSION_SCHEMES
+    )
     if file_exists:
         if ext == '' or ext not in all_valid_exts:
-            raise EnvironmentFileExtensionNotValid(filename)
+            raise EnvironmentFileExtensionNotValid(filename or None)
         elif ext in YamlFileSpec.extensions:
             specs = [YamlFileSpec]
         elif ext in RequirementsSpec.extensions:
@@ -41,7 +44,7 @@ def detect(**kwargs):
             return spec
 
     if not file_exists and remote_definition is None:
-        raise EnvironmentFileNotFound(filename=filename)
+        raise EnvironmentFileNotFound(filename=filename or None)
     else:
         raise SpecNotFound(build_message(spec_instances))
 

tests/conda_env/test_cli.py

@@ -244,6 +244,22 @@ def test_create_valid_env(self):
             len([env for env in parsed['envs'] if env.endswith(test_env_name_1)]), 0
         )
 
+    @pytest.mark.integration
+    def test_conda_env_create_http(self):
+        '''
+        Test `conda env create --file=https://some-website.com/environment.yml`
+        '''
+        run_env_command(
+            Commands.ENV_CREATE,
+            None,
+            '--file',
+            'https://raw.githubusercontent.com/conda/conda/master/tests/conda_env/support/simple.yml',
+        )
+        try:
+            self.assertTrue(env_is_created("nlp"))
+        finally:
+            run_env_command(Commands.ENV_REMOVE, "nlp")
+
     def test_update(self):
         create_env(environment_1)
         run_env_command(Commands.ENV_CREATE, None)

tests/conda_env/test_env.py

@@ -7,6 +7,7 @@
 
 from conda.core.prefix_data import PrefixData
 from conda.base.context import conda_tests_ctxt_mgmt_def_pol
+from conda.exceptions import CondaHTTPError
 from conda.models.match_spec import MatchSpec
 from conda.common.io import env_vars
 from conda.common.serialize import yaml_load
@@ -77,6 +78,18 @@ def test_with_pip(self):
         assert 'foo' in e.dependencies['pip']
         assert 'baz' in e.dependencies['pip']
 
+    @pytest.mark.integration
+    def test_http(self):
+        e = get_simple_environment()
+        f = env.from_file("https://raw.githubusercontent.com/conda/conda/master/tests/conda_env/support/simple.yml")
+        self.assertEqual(e.dependencies, f.dependencies)
+        assert e.dependencies == f.dependencies
+
+    @pytest.mark.integration
+    def test_http_raises(self):
+        with self.assertRaises(CondaHTTPError):
+            env.from_file("https://raw.githubusercontent.com/conda/conda/master/tests/conda_env/support/does-not-exist.yml")
+
 
 class EnvironmentTestCase(unittest.TestCase):
     def test_has_empty_filename_by_default(self):