Add json output to conda env create/update (conda#9204)

Add json output to conda env create/update

conda_env/cli/common.py

@@ -7,6 +7,8 @@
 
 from conda._vendor.auxlib.entity import EntityEncoder
 from conda.base.context import context
+from conda.cli import install as cli_install
+from conda.cli import common as cli_common
 
 base_env_name = 'base'
 
@@ -32,3 +34,19 @@ def find_prefix_name(name):
         if isdir(prefix):
             return prefix
     return None
+
+
+def print_result(args, prefix, result):
+    if context.json:
+        if result["conda"] is None and result["pip"] is None:
+            cli_common.stdout_json_success(message='All requested packages already installed.')
+        else:
+            if result["conda"] is not None:
+                actions = result["conda"]
+            else:
+                actions = {}
+            if result["pip"] is not None:
+                actions["PIP"] = result["pip"]
+            cli_common.stdout_json_success(prefix=prefix, actions=actions)
+    else:
+        cli_install.print_activate(args.name if args.name else prefix)

conda_env/cli/main.py

@@ -6,6 +6,11 @@
 import os
 import sys
 
+# pip_util.py import on_win from conda.exports
+# conda.exports resets the context
+# we need to import conda.exports here so that the context is not lost
+# when importing pip (and pip_util)
+import conda.exports  # noqa
 from conda.base.context import context
 from conda.cli.conda_argparse import ArgumentParser
 from conda.cli.main import init_loggers

conda_env/cli/main_create.py

@@ -13,7 +13,7 @@
 from conda.cli.conda_argparse import add_parser_json, add_parser_prefix
 from conda.gateways.disk.delete import rm_rf
 from conda.misc import touch_nonadmin
-from .common import get_prefix
+from .common import get_prefix, print_result
 from .. import exceptions, specs
 from ..installers.base import InvalidInstaller, get_installer
 
@@ -95,10 +95,11 @@ def execute(args, parser):
     # common.ensure_override_channels_requires_channel(args)
     # channel_urls = args.channel or ()
 
+    result = {"conda": None, "pip": None}
     for installer_type, pkg_specs in env.dependencies.items():
         try:
             installer = get_installer(installer_type)
-            installer.install(prefix, pkg_specs, args, env)
+            result[installer_type] = installer.install(prefix, pkg_specs, args, env)
         except InvalidInstaller:
             sys.stderr.write(textwrap.dedent("""
                 Unable to install package for {0}.
@@ -112,5 +113,4 @@ def execute(args, parser):
             return -1
 
     touch_nonadmin(prefix)
-    if not args.json:
-        cli_install.print_activate(args.name if args.name else prefix)
+    print_result(args, prefix, result)

conda_env/cli/main_update.py

@@ -7,10 +7,9 @@
 import textwrap
 
 from conda._vendor.auxlib.path import expand
-from conda.cli import install as cli_install
 from conda.cli.conda_argparse import add_parser_json, add_parser_prefix
 from conda.misc import touch_nonadmin
-from .common import get_prefix
+from .common import get_prefix, print_result
 from .. import exceptions, specs as install_specs
 from ..exceptions import CondaEnvException
 from ..installers.base import InvalidInstaller, get_installer
@@ -119,9 +118,10 @@ def execute(args, parser):
             )
             return -1
 
+    result = {"conda": None, "pip": None}
     for installer_type, specs in env.dependencies.items():
         installer = installers[installer_type]
-        installer.install(prefix, specs, args, env)
+        result[installer_type] = installer.install(prefix, specs, args, env)
 
     touch_nonadmin(prefix)
-    cli_install.print_activate(args.name if args.name else prefix)
+    print_result(args, prefix, result)

conda_env/installers/conda.py

@@ -26,6 +26,8 @@ def install(prefix, specs, args, env, *_, **kwargs):
     solver = Solver(prefix, channels, subdirs, specs_to_add=specs)
     unlink_link_transaction = solver.solve_for_transaction(prune=getattr(args, 'prune', False))
 
-    pfe = unlink_link_transaction._get_pfe()
-    pfe.execute()
+    if unlink_link_transaction.nothing_to_do:
+        return None
+    unlink_link_transaction.download_and_extract()
     unlink_link_transaction.execute()
+    return unlink_link_transaction._make_legacy_action_groups()[0]

conda_env/installers/pip.py

@@ -6,7 +6,7 @@
 import os
 import os.path as op
 from conda._vendor.auxlib.compat import Utf8NamedTemporaryFile
-from conda_env.pip_util import pip_subprocess
+from conda_env.pip_util import pip_subprocess, get_pip_installed_packages
 from logging import getLogger
 
 
@@ -43,7 +43,7 @@ def _pip_install_via_requirements(prefix, specs, args, *_, **kwargs):
         requirements.close()
         # pip command line...
         pip_cmd = ['install', '-U', '-r', requirements.name]
-        pip_subprocess(pip_cmd, prefix, cwd=pip_workdir)
+        stdout, stderr = pip_subprocess(pip_cmd, prefix, cwd=pip_workdir)
     finally:
         # Win/Appveyor does not like it if we use context manager + delete=True.
         # So we delete the temporary file in a finally block.
@@ -53,6 +53,7 @@ def _pip_install_via_requirements(prefix, specs, args, *_, **kwargs):
             else:
                 log.warning('CONDA_TEST_SAVE_TEMPS :: retaining pip requirements.txt {}'
                             .format(requirements.name))
+    return get_pip_installed_packages(stdout)
 
 
 # Conform to Installers API

conda_env/pip_util.py

@@ -12,10 +12,12 @@
 from logging import getLogger
 import os
 import re
+import sys
 
 from .exceptions import CondaEnvException
 from conda.gateways.subprocess import any_subprocess
 from conda.exports import on_win
+from conda.base.context import context
 
 
 log = getLogger(__name__)
@@ -28,17 +30,28 @@ def pip_subprocess(args, prefix, cwd):
         python_path = os.path.join(prefix, 'bin', 'python')
     run_args = [python_path, '-m', 'pip'] + args
     stdout, stderr, rc = any_subprocess(run_args, prefix, cwd=cwd)
-    print("Ran pip subprocess with arguments:")
-    print(run_args)
-    print("Pip subprocess output:")
-    print(stdout)
+    if not context.quiet and not context.json:
+        print("Ran pip subprocess with arguments:")
+        print(run_args)
+        print("Pip subprocess output:")
+        print(stdout)
     if rc != 0:
-        print("Pip subprocess error:")
-        print(stderr)
+        print("Pip subprocess error:", file=sys.stderr)
+        print(stderr, file=sys.stderr)
         raise CondaEnvException("Pip failed")
 
     # This will modify (break) Context. We have a context stack but need to verify it works
     # stdout, stderr, rc = run_command(Commands.RUN, *run_args, stdout=None, stderr=None)
+    return stdout, stderr
+
+
+def get_pip_installed_packages(stdout):
+    """Return the list of pip packages installed based on the command output"""
+    m = re.search(r"Successfully installed\ (.*)", stdout)
+    if m:
+        return m.group(1).strip().split()
+    else:
+        return None
 
 
 def get_pip_version(prefix):
@@ -75,7 +88,7 @@ def installed(prefix, output=True):
     except Exception:
         # Any error should just be ignored
         if output:
-            print("# Warning: subprocess call to pip failed")
+            print("# Warning: subprocess call to pip failed", file=sys.stderr)
         return
 
     if pip_major_version >= 9:
@@ -111,7 +124,7 @@ def installed(prefix, output=True):
             m = pat.match(line)
             if m is None:
                 if output:
-                    print('Could not extract name and version from: %r' % line)
+                    print('Could not extract name and version from: %r' % line, file=sys.stderr)
                 continue
             name, version = m.groups()
             name = name.lower()

tests/conda_env/test_cli.py

@@ -47,6 +47,29 @@
 foo: bar
 '''
 
+environment_python_pip_click = '''
+name: env-1
+dependencies:
+  - python=3
+  - pip
+  - pip:
+    - click
+channels:
+  - defaults
+'''
+
+environment_python_pip_click_attrs = '''
+name: env-1
+dependencies:
+  - python=3
+  - pip
+  - pip:
+    - click
+    - attrs
+channels:
+  - defaults
+'''
+
 test_env_name_1 = "env-1"
 test_env_name_2 = "snowflakes"
 test_env_name_42 = "env-42"
@@ -147,11 +170,15 @@ def setUp(self):
         rm_rf("environment.yml")
         run_env_command(Commands.ENV_REMOVE, test_env_name_1)
         run_env_command(Commands.ENV_REMOVE, test_env_name_42)
+        for env_nb in range(1, 6):
+            run_env_command(Commands.ENV_REMOVE, "envjson-{0}".format(env_nb))
 
     def tearDown(self):
         rm_rf("environment.yml")
         run_env_command(Commands.ENV_REMOVE, test_env_name_1)
         run_env_command(Commands.ENV_REMOVE, test_env_name_42)
+        for env_nb in range(1, 6):
+             run_env_command(Commands.ENV_REMOVE, "envjson-{0}".format(env_nb))
 
     def test_conda_env_create_no_file(self):
         '''
@@ -238,6 +265,71 @@ def test_name(self):
             len([env for env in parsed['envs'] if env.endswith(env_name)]), 0
         )
 
+    def test_create_valid_env_json_output(self):
+        """
+        Creates an environment from an environment.yml file with conda packages (no pip)
+        Check the json output
+        """
+        create_env(environment_1)
+        stdout, stderr = run_env_command(Commands.ENV_CREATE, "envjson-1", "--quiet", "--json")
+        output = json.loads(stdout)
+        assert output["success"] is True
+        assert len(output["actions"]["LINK"]) > 0
+        assert "PIP" not in output["actions"]
+
+    def test_create_valid_env_with_conda_and_pip_json_output(self):
+        """
+        Creates an environment from an environment.yml file with conda and pip dependencies
+        Check the json output
+        """
+        create_env(environment_python_pip_click)
+        stdout, stderr = run_env_command(Commands.ENV_CREATE, "envjson-2", "--quiet", "--json")
+        output = json.loads(stdout)
+        assert len(output["actions"]["LINK"]) > 0
+        assert output["actions"]["PIP"][0].startswith("click")
+
+    def test_update_env_json_output(self):
+        """
+        Update an environment by adding a conda package
+        Check the json output
+        """
+        create_env(environment_1)
+        run_env_command(Commands.ENV_CREATE, "envjson-3", "--json")
+        create_env(environment_2)
+        stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-3", "--quiet", "--json")
+        output = json.loads(stdout)
+        assert output["success"] is True
+        assert len(output["actions"]["LINK"]) > 0
+        assert "PIP" not in output["actions"]
+
+    def test_update_env_only_pip_json_output(self):
+        """
+        Update an environment by adding only a pip package
+        Check the json output
+        """
+        create_env(environment_python_pip_click)
+        run_env_command(Commands.ENV_CREATE, "envjson-4", "--json")
+        create_env(environment_python_pip_click_attrs)
+        stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-4", "--quiet", "--json")
+        output = json.loads(stdout)
+        assert output["success"] is True
+        # No conda actions (FETCH/LINK), only pip
+        assert list(output["actions"].keys()) == ["PIP"]
+        # Only attrs installed
+        assert len(output["actions"]["PIP"]) == 1
+        assert output["actions"]["PIP"][0].startswith("attrs")
+
+    def test_update_env_no_action_json_output(self):
+        """
+        Update an already up-to-date environment
+        Check the json output
+        """
+        create_env(environment_python_pip_click)
+        run_env_command(Commands.ENV_CREATE, "envjson-5", "--json")
+        stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-5", "--quiet", "--json")
+        output = json.loads(stdout)
+        assert output["message"] == "All requested packages already installed."
+
 
 def env_is_created(env_name):
     """

tests/conda_env/test_pip_util.py

@@ -0,0 +1,77 @@
+import pytest
+from conda_env.pip_util import get_pip_installed_packages
+
+pip_output_attrs = """
+Collecting attrs
+  Using cached https://files.pythonhosted.org/packages/23/96/d828354fa2dbdf216eaa7b7de0db692f12c234f7ef888cc14980ef40d1d2/attrs-19.1.0-py2.py3-none-any.whl
+Installing collected packages: attrs
+Successfully installed attrs-19.1.0
+"""
+pip_output_attrs_expected = ["attrs-19.1.0"]
+
+pip_output_flask = """
+Collecting flask
+  Using cached https://files.pythonhosted.org/packages/9b/93/628509b8d5dc749656a9641f4caf13540e2cdec85276964ff8f43bbb1d3b/Flask-1.1.1-py2.py3-none-any.whl
+Collecting itsdangerous>=0.24 (from flask)
+  Using cached https://files.pythonhosted.org/packages/76/ae/44b03b253d6fade317f32c24d100b3b35c2239807046a4c953c7b89fa49e/itsdangerous-1.1.0-py2.py3-none-any.whl
+Collecting click>=5.1 (from flask)
+  Using cached https://files.pythonhosted.org/packages/fa/37/45185cb5abbc30d7257104c434fe0b07e5a195a6847506c074527aa599ec/Click-7.0-py2.py3-none-any.whl
+Collecting Werkzeug>=0.15 (from flask)
+  Using cached https://files.pythonhosted.org/packages/b7/61/c0a1adf9ad80db012ed7191af98fa05faa95fa09eceb71bb6fa8b66e6a43/Werkzeug-0.15.6-py2.py3-none-any.whl
+Collecting Jinja2>=2.10.1 (from flask)
+  Using cached https://files.pythonhosted.org/packages/1d/e7/fd8b501e7a6dfe492a433deb7b9d833d39ca74916fa8bc63dd1a4947a671/Jinja2-2.10.1-py2.py3-none-any.whl
+Collecting MarkupSafe>=0.23 (from Jinja2>=2.10.1->flask)
+  Using cached https://files.pythonhosted.org/packages/ce/c6/f000f1af136ef74e4a95e33785921c73595c5390403f102e9b231b065b7a/MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl
+Installing collected packages: itsdangerous, click, Werkzeug, MarkupSafe, Jinja2, flask
+Successfully installed Jinja2-2.10.1 MarkupSafe-1.1.1 Werkzeug-0.15.6 click-7.0 flask-1.1.1 itsdangerous-1.1.0
+"""
+pip_output_flask_expected = [
+    "Jinja2-2.10.1",
+    "MarkupSafe-1.1.1",
+    "Werkzeug-0.15.6",
+    "click-7.0",
+    "flask-1.1.1",
+    "itsdangerous-1.1.0",
+]
+
+pip_output_flask_only = """
+Collecting flask
+  Using cached https://files.pythonhosted.org/packages/9b/93/628509b8d5dc749656a9641f4caf13540e2cdec85276964ff8f43bbb1d3b/Flask-1.1.1-py2.py3-none-any.whl
+Requirement already satisfied: Werkzeug>=0.15 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (0.15.6)
+Requirement already satisfied: itsdangerous>=0.24 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (1.1.0)
+Requirement already satisfied: Jinja2>=2.10.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (2.10.1)
+Requirement already satisfied: click>=5.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (7.0)
+Requirement already satisfied: MarkupSafe>=0.23 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from Jinja2>=2.10.1->flask) (1.1.1)
+Installing collected packages: flask
+Successfully installed flask-1.1.1
+"""
+pip_output_flask_only_expected = ["flask-1.1.1"]
+
+pip_output_flask_already_installed = """
+Requirement already satisfied: flask in ./miniconda3/envs/fooo/lib/python3.7/site-packages (1.1.1)
+Requirement already satisfied: itsdangerous>=0.24 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (1.1.0)
+Requirement already satisfied: Jinja2>=2.10.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (2.10.1)
+Requirement already satisfied: click>=5.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (7.0)
+Requirement already satisfied: Werkzeug>=0.15 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (0.15.6)
+Requirement already satisfied: MarkupSafe>=0.23 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from Jinja2>=2.10.1->flask) (1.1.1)
+"""
+
+
+@pytest.mark.parametrize(
+    "pip_output,expected",
+    [
+        ("Successfully installed foo bar", ["foo", "bar"]),
+        (pip_output_attrs, pip_output_attrs_expected),
+        (pip_output_flask, pip_output_flask_expected),
+        (pip_output_flask_only, pip_output_flask_only_expected),
+    ],
+)
+def test_get_pip_installed_packages(pip_output, expected):
+    result = get_pip_installed_packages(pip_output)
+    assert result == expected
+
+
+@pytest.mark.parametrize("pip_output", [pip_output_flask_already_installed, "foo", ""])
+def test_get_pip_installed_packages_none(pip_output):
+    result = get_pip_installed_packages(pip_output)
+    assert result is None