From 12dbb79b62163894f0d12dbe6462fe8bb44bb00f Mon Sep 17 00:00:00 2001 From: Benjamin Bertrand Date: Wed, 4 Sep 2019 16:26:32 +0200 Subject: [PATCH] Add json output to conda env create/update --- conda_env/cli/common.py | 18 +++++++ conda_env/cli/main.py | 5 ++ conda_env/cli/main_create.py | 8 +-- conda_env/cli/main_update.py | 8 +-- conda_env/installers/conda.py | 6 ++- conda_env/installers/pip.py | 5 +- conda_env/pip_util.py | 29 +++++++--- tests/conda_env/test_cli.py | 92 ++++++++++++++++++++++++++++++++ tests/conda_env/test_pip_util.py | 77 ++++++++++++++++++++++++++ 9 files changed, 228 insertions(+), 20 deletions(-) create mode 100644 tests/conda_env/test_pip_util.py diff --git a/conda_env/cli/common.py b/conda_env/cli/common.py index ea1105a455d..ea92fe01b66 100644 --- a/conda_env/cli/common.py +++ b/conda_env/cli/common.py @@ -7,6 +7,8 @@ from conda._vendor.auxlib.entity import EntityEncoder from conda.base.context import context +from conda.cli import install as cli_install +from conda.cli import common as cli_common base_env_name = 'base' @@ -32,3 +34,19 @@ def find_prefix_name(name): if isdir(prefix): return prefix return None + + +def print_result(args, prefix, result): + if context.json: + if result["conda"] is None and result["pip"] is None: + cli_common.stdout_json_success(message='All requested packages already installed.') + else: + if result["conda"] is not None: + actions = result["conda"] + else: + actions = {} + if result["pip"] is not None: + actions["PIP"] = result["pip"] + cli_common.stdout_json_success(prefix=prefix, actions=actions) + else: + cli_install.print_activate(args.name if args.name else prefix) diff --git a/conda_env/cli/main.py b/conda_env/cli/main.py index 7525e4d03db..20b0c9ae0ad 100644 --- a/conda_env/cli/main.py +++ b/conda_env/cli/main.py @@ -6,6 +6,11 @@ import os import sys +# pip_util.py import on_win from conda.exports +# conda.exports resets the context +# we need to import conda.exports here so that the context is not lost +# when importing pip (and pip_util) +import conda.exports # noqa from conda.base.context import context from conda.cli.conda_argparse import ArgumentParser from conda.cli.main import init_loggers diff --git a/conda_env/cli/main_create.py b/conda_env/cli/main_create.py index 54e2e029523..3ecd218738c 100644 --- a/conda_env/cli/main_create.py +++ b/conda_env/cli/main_create.py @@ -13,7 +13,7 @@ from conda.cli.conda_argparse import add_parser_json, add_parser_prefix from conda.gateways.disk.delete import rm_rf from conda.misc import touch_nonadmin -from .common import get_prefix +from .common import get_prefix, print_result from .. import exceptions, specs from ..installers.base import InvalidInstaller, get_installer @@ -95,10 +95,11 @@ def execute(args, parser): # common.ensure_override_channels_requires_channel(args) # channel_urls = args.channel or () + result = {"conda": None, "pip": None} for installer_type, pkg_specs in env.dependencies.items(): try: installer = get_installer(installer_type) - installer.install(prefix, pkg_specs, args, env) + result[installer_type] = installer.install(prefix, pkg_specs, args, env) except InvalidInstaller: sys.stderr.write(textwrap.dedent(""" Unable to install package for {0}. @@ -112,5 +113,4 @@ def execute(args, parser): return -1 touch_nonadmin(prefix) - if not args.json: - cli_install.print_activate(args.name if args.name else prefix) + print_result(args, prefix, result) diff --git a/conda_env/cli/main_update.py b/conda_env/cli/main_update.py index c613fe435f7..fe1160f7b94 100644 --- a/conda_env/cli/main_update.py +++ b/conda_env/cli/main_update.py @@ -7,10 +7,9 @@ import textwrap from conda._vendor.auxlib.path import expand -from conda.cli import install as cli_install from conda.cli.conda_argparse import add_parser_json, add_parser_prefix from conda.misc import touch_nonadmin -from .common import get_prefix +from .common import get_prefix, print_result from .. import exceptions, specs as install_specs from ..exceptions import CondaEnvException from ..installers.base import InvalidInstaller, get_installer @@ -119,9 +118,10 @@ def execute(args, parser): ) return -1 + result = {"conda": None, "pip": None} for installer_type, specs in env.dependencies.items(): installer = installers[installer_type] - installer.install(prefix, specs, args, env) + result[installer_type] = installer.install(prefix, specs, args, env) touch_nonadmin(prefix) - cli_install.print_activate(args.name if args.name else prefix) + print_result(args, prefix, result) diff --git a/conda_env/installers/conda.py b/conda_env/installers/conda.py index 079f0ee44e1..0eb3ce829cd 100644 --- a/conda_env/installers/conda.py +++ b/conda_env/installers/conda.py @@ -26,6 +26,8 @@ def install(prefix, specs, args, env, *_, **kwargs): solver = Solver(prefix, channels, subdirs, specs_to_add=specs) unlink_link_transaction = solver.solve_for_transaction(prune=getattr(args, 'prune', False)) - pfe = unlink_link_transaction._get_pfe() - pfe.execute() + if unlink_link_transaction.nothing_to_do: + return None + unlink_link_transaction.download_and_extract() unlink_link_transaction.execute() + return unlink_link_transaction._make_legacy_action_groups()[0] diff --git a/conda_env/installers/pip.py b/conda_env/installers/pip.py index c020138721b..ea13b111c2a 100644 --- a/conda_env/installers/pip.py +++ b/conda_env/installers/pip.py @@ -6,7 +6,7 @@ import os import os.path as op from conda._vendor.auxlib.compat import Utf8NamedTemporaryFile -from conda_env.pip_util import pip_subprocess +from conda_env.pip_util import pip_subprocess, get_pip_installed_packages from logging import getLogger @@ -43,7 +43,7 @@ def _pip_install_via_requirements(prefix, specs, args, *_, **kwargs): requirements.close() # pip command line... pip_cmd = ['install', '-U', '-r', requirements.name] - pip_subprocess(pip_cmd, prefix, cwd=pip_workdir) + stdout, stderr = pip_subprocess(pip_cmd, prefix, cwd=pip_workdir) finally: # Win/Appveyor does not like it if we use context manager + delete=True. # So we delete the temporary file in a finally block. @@ -53,6 +53,7 @@ def _pip_install_via_requirements(prefix, specs, args, *_, **kwargs): else: log.warning('CONDA_TEST_SAVE_TEMPS :: retaining pip requirements.txt {}' .format(requirements.name)) + return get_pip_installed_packages(stdout) # Conform to Installers API diff --git a/conda_env/pip_util.py b/conda_env/pip_util.py index 0f0da2d6510..eac9dad3999 100644 --- a/conda_env/pip_util.py +++ b/conda_env/pip_util.py @@ -12,10 +12,12 @@ from logging import getLogger import os import re +import sys from .exceptions import CondaEnvException from conda.gateways.subprocess import any_subprocess from conda.exports import on_win +from conda.base.context import context log = getLogger(__name__) @@ -28,17 +30,28 @@ def pip_subprocess(args, prefix, cwd): python_path = os.path.join(prefix, 'bin', 'python') run_args = [python_path, '-m', 'pip'] + args stdout, stderr, rc = any_subprocess(run_args, prefix, cwd=cwd) - print("Ran pip subprocess with arguments:") - print(run_args) - print("Pip subprocess output:") - print(stdout) + if not context.quiet and not context.json: + print("Ran pip subprocess with arguments:") + print(run_args) + print("Pip subprocess output:") + print(stdout) if rc != 0: - print("Pip subprocess error:") - print(stderr) + print("Pip subprocess error:", file=sys.stderr) + print(stderr, file=sys.stderr) raise CondaEnvException("Pip failed") # This will modify (break) Context. We have a context stack but need to verify it works # stdout, stderr, rc = run_command(Commands.RUN, *run_args, stdout=None, stderr=None) + return stdout, stderr + + +def get_pip_installed_packages(stdout): + """Return the list of pip packages installed based on the command output""" + m = re.search(r"Successfully installed\ (.*)", stdout) + if m: + return m.group(1).strip().split() + else: + return None def get_pip_version(prefix): @@ -75,7 +88,7 @@ def installed(prefix, output=True): except Exception: # Any error should just be ignored if output: - print("# Warning: subprocess call to pip failed") + print("# Warning: subprocess call to pip failed", file=sys.stderr) return if pip_major_version >= 9: @@ -111,7 +124,7 @@ def installed(prefix, output=True): m = pat.match(line) if m is None: if output: - print('Could not extract name and version from: %r' % line) + print('Could not extract name and version from: %r' % line, file=sys.stderr) continue name, version = m.groups() name = name.lower() diff --git a/tests/conda_env/test_cli.py b/tests/conda_env/test_cli.py index a1129f6ef38..00dc6cf90e4 100644 --- a/tests/conda_env/test_cli.py +++ b/tests/conda_env/test_cli.py @@ -47,6 +47,29 @@ foo: bar ''' +environment_python_pip_click = ''' +name: env-1 +dependencies: + - python=3 + - pip + - pip: + - click +channels: + - defaults +''' + +environment_python_pip_click_attrs = ''' +name: env-1 +dependencies: + - python=3 + - pip + - pip: + - click + - attrs +channels: + - defaults +''' + test_env_name_1 = "env-1" test_env_name_2 = "snowflakes" test_env_name_42 = "env-42" @@ -147,11 +170,15 @@ def setUp(self): rm_rf("environment.yml") run_env_command(Commands.ENV_REMOVE, test_env_name_1) run_env_command(Commands.ENV_REMOVE, test_env_name_42) + for env_nb in range(1, 6): + run_env_command(Commands.ENV_REMOVE, "envjson-{0}".format(env_nb)) def tearDown(self): rm_rf("environment.yml") run_env_command(Commands.ENV_REMOVE, test_env_name_1) run_env_command(Commands.ENV_REMOVE, test_env_name_42) + for env_nb in range(1, 6): + run_env_command(Commands.ENV_REMOVE, "envjson-{0}".format(env_nb)) def test_conda_env_create_no_file(self): ''' @@ -238,6 +265,71 @@ def test_name(self): len([env for env in parsed['envs'] if env.endswith(env_name)]), 0 ) + def test_create_valid_env_json_output(self): + """ + Creates an environment from an environment.yml file with conda packages (no pip) + Check the json output + """ + create_env(environment_1) + stdout, stderr = run_env_command(Commands.ENV_CREATE, "envjson-1", "--quiet", "--json") + output = json.loads(stdout) + assert output["success"] is True + assert len(output["actions"]["LINK"]) > 0 + assert "PIP" not in output["actions"] + + def test_create_valid_env_with_conda_and_pip_json_output(self): + """ + Creates an environment from an environment.yml file with conda and pip dependencies + Check the json output + """ + create_env(environment_python_pip_click) + stdout, stderr = run_env_command(Commands.ENV_CREATE, "envjson-2", "--quiet", "--json") + output = json.loads(stdout) + assert len(output["actions"]["LINK"]) > 0 + assert output["actions"]["PIP"][0].startswith("click") + + def test_update_env_json_output(self): + """ + Update an environment by adding a conda package + Check the json output + """ + create_env(environment_1) + run_env_command(Commands.ENV_CREATE, "envjson-3", "--json") + create_env(environment_2) + stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-3", "--quiet", "--json") + output = json.loads(stdout) + assert output["success"] is True + assert len(output["actions"]["LINK"]) > 0 + assert "PIP" not in output["actions"] + + def test_update_env_only_pip_json_output(self): + """ + Update an environment by adding only a pip package + Check the json output + """ + create_env(environment_python_pip_click) + run_env_command(Commands.ENV_CREATE, "envjson-4", "--json") + create_env(environment_python_pip_click_attrs) + stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-4", "--quiet", "--json") + output = json.loads(stdout) + assert output["success"] is True + # No conda actions (FETCH/LINK), only pip + assert list(output["actions"].keys()) == ["PIP"] + # Only attrs installed + assert len(output["actions"]["PIP"]) == 1 + assert output["actions"]["PIP"][0].startswith("attrs") + + def test_update_env_no_action_json_output(self): + """ + Update an already up-to-date environment + Check the json output + """ + create_env(environment_python_pip_click) + run_env_command(Commands.ENV_CREATE, "envjson-5", "--json") + stdout, stderr = run_env_command(Commands.ENV_UPDATE, "envjson-5", "--quiet", "--json") + output = json.loads(stdout) + assert output["message"] == "All requested packages already installed." + def env_is_created(env_name): """ diff --git a/tests/conda_env/test_pip_util.py b/tests/conda_env/test_pip_util.py new file mode 100644 index 00000000000..be372d59bd9 --- /dev/null +++ b/tests/conda_env/test_pip_util.py @@ -0,0 +1,77 @@ +import pytest +from conda_env.pip_util import get_pip_installed_packages + +pip_output_attrs = """ +Collecting attrs + Using cached https://files.pythonhosted.org/packages/23/96/d828354fa2dbdf216eaa7b7de0db692f12c234f7ef888cc14980ef40d1d2/attrs-19.1.0-py2.py3-none-any.whl +Installing collected packages: attrs +Successfully installed attrs-19.1.0 +""" +pip_output_attrs_expected = ["attrs-19.1.0"] + +pip_output_flask = """ +Collecting flask + Using cached https://files.pythonhosted.org/packages/9b/93/628509b8d5dc749656a9641f4caf13540e2cdec85276964ff8f43bbb1d3b/Flask-1.1.1-py2.py3-none-any.whl +Collecting itsdangerous>=0.24 (from flask) + Using cached https://files.pythonhosted.org/packages/76/ae/44b03b253d6fade317f32c24d100b3b35c2239807046a4c953c7b89fa49e/itsdangerous-1.1.0-py2.py3-none-any.whl +Collecting click>=5.1 (from flask) + Using cached https://files.pythonhosted.org/packages/fa/37/45185cb5abbc30d7257104c434fe0b07e5a195a6847506c074527aa599ec/Click-7.0-py2.py3-none-any.whl +Collecting Werkzeug>=0.15 (from flask) + Using cached https://files.pythonhosted.org/packages/b7/61/c0a1adf9ad80db012ed7191af98fa05faa95fa09eceb71bb6fa8b66e6a43/Werkzeug-0.15.6-py2.py3-none-any.whl +Collecting Jinja2>=2.10.1 (from flask) + Using cached https://files.pythonhosted.org/packages/1d/e7/fd8b501e7a6dfe492a433deb7b9d833d39ca74916fa8bc63dd1a4947a671/Jinja2-2.10.1-py2.py3-none-any.whl +Collecting MarkupSafe>=0.23 (from Jinja2>=2.10.1->flask) + Using cached https://files.pythonhosted.org/packages/ce/c6/f000f1af136ef74e4a95e33785921c73595c5390403f102e9b231b065b7a/MarkupSafe-1.1.1-cp37-cp37m-macosx_10_6_intel.whl +Installing collected packages: itsdangerous, click, Werkzeug, MarkupSafe, Jinja2, flask +Successfully installed Jinja2-2.10.1 MarkupSafe-1.1.1 Werkzeug-0.15.6 click-7.0 flask-1.1.1 itsdangerous-1.1.0 +""" +pip_output_flask_expected = [ + "Jinja2-2.10.1", + "MarkupSafe-1.1.1", + "Werkzeug-0.15.6", + "click-7.0", + "flask-1.1.1", + "itsdangerous-1.1.0", +] + +pip_output_flask_only = """ +Collecting flask + Using cached https://files.pythonhosted.org/packages/9b/93/628509b8d5dc749656a9641f4caf13540e2cdec85276964ff8f43bbb1d3b/Flask-1.1.1-py2.py3-none-any.whl +Requirement already satisfied: Werkzeug>=0.15 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (0.15.6) +Requirement already satisfied: itsdangerous>=0.24 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (1.1.0) +Requirement already satisfied: Jinja2>=2.10.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (2.10.1) +Requirement already satisfied: click>=5.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (7.0) +Requirement already satisfied: MarkupSafe>=0.23 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from Jinja2>=2.10.1->flask) (1.1.1) +Installing collected packages: flask +Successfully installed flask-1.1.1 +""" +pip_output_flask_only_expected = ["flask-1.1.1"] + +pip_output_flask_already_installed = """ +Requirement already satisfied: flask in ./miniconda3/envs/fooo/lib/python3.7/site-packages (1.1.1) +Requirement already satisfied: itsdangerous>=0.24 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (1.1.0) +Requirement already satisfied: Jinja2>=2.10.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (2.10.1) +Requirement already satisfied: click>=5.1 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (7.0) +Requirement already satisfied: Werkzeug>=0.15 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from flask) (0.15.6) +Requirement already satisfied: MarkupSafe>=0.23 in ./miniconda3/envs/fooo/lib/python3.7/site-packages (from Jinja2>=2.10.1->flask) (1.1.1) +""" + + +@pytest.mark.parametrize( + "pip_output,expected", + [ + ("Successfully installed foo bar", ["foo", "bar"]), + (pip_output_attrs, pip_output_attrs_expected), + (pip_output_flask, pip_output_flask_expected), + (pip_output_flask_only, pip_output_flask_only_expected), + ], +) +def test_get_pip_installed_packages(pip_output, expected): + result = get_pip_installed_packages(pip_output) + assert result == expected + + +@pytest.mark.parametrize("pip_output", [pip_output_flask_already_installed, "foo", ""]) +def test_get_pip_installed_packages_none(pip_output): + result = get_pip_installed_packages(pip_output) + assert result is None