diff --git a/conda/core/prefix_data.py b/conda/core/prefix_data.py index ecc7ce6727a..7fa496380bb 100644 --- a/conda/core/prefix_data.py +++ b/conda/core/prefix_data.py @@ -61,7 +61,7 @@ def __init__(self, prefix_path, pip_interop_enabled=None): if pip_interop_enabled is not None else context.pip_interop_enabled) - @time_recorder + @time_recorder(module_name=__name__) def load(self): self.__prefix_records = {} _conda_meta_dir = join(self.prefix_path, 'conda-meta') diff --git a/conda/core/solve.py b/conda/core/solve.py index b6a18d4b319..79367586f06 100644 --- a/conda/core/solve.py +++ b/conda/core/solve.py @@ -3,6 +3,8 @@ # SPDX-License-Identifier: BSD-3-Clause from __future__ import absolute_import, division, print_function, unicode_literals +from collections import deque +import copy from genericpath import exists from logging import DEBUG, getLogger from os.path import join @@ -257,18 +259,14 @@ def solve_final_state(self, update_modifier=NULL, deps_modifier=NULL, prune=NULL @time_recorder(module_name=__name__) def _collect_all_metadata(self, ssc): - if ssc.prune: # or update_modifier == UpdateModifier.UPDATE_ALL # pending conda/constructor#138 # NOQA - # Users are struggling with the prune functionality in --update-all, due to - # https://github.com/conda/constructor/issues/138. Until that issue is resolved, - # and for the foreseeable future, it's best to be more conservative with --update-all. - + if ssc.specs_from_history_map: # Start with empty specs map for UPDATE_ALL because we're optimizing the update # only for specs the user has requested; it's ok to remove dependencies. # However, because of https://github.com/conda/constructor/issues/138, we need # to hard-code keeping conda, conda-build, and anaconda, if they're already in # the environment. - for pkg_name in ('anaconda', 'conda', 'conda-build'): + for pkg_name in ('anaconda', 'conda', 'conda-build', 'python'): if ssc.prefix_data.get(pkg_name, None): ssc.specs_from_history_map[pkg_name] = MatchSpec(pkg_name) else: @@ -279,6 +277,12 @@ def _collect_all_metadata(self, ssc): # add in historically-requested specs ssc.specs_map.update(ssc.specs_from_history_map) + # add in aggressively updated packages + ssc.specs_map.update( + (prec.name, MatchSpec(prec.name)) for prec in ssc.prefix_data.iter_records() + if MatchSpec(prec.name) in context.aggressive_update_packages + ) + prepared_specs = set(concatv( self.specs_to_remove, self.specs_to_add, @@ -383,6 +387,63 @@ def _add_specs(self, ssc): # TLDR: when working with MatchSpec objects, # - to minimize the version change, set MatchSpec(name=name, target=prec.dist_str()) # - to freeze the package, set all the components of MatchSpec individually + + # the only things we should consider freezing are things that don't conflict with the new + # specs being added. + explicit_spec_package_pool = {} + if ssc.update_modifier == UpdateModifier.FREEZE_INSTALLED: + specs_seen = set() + specs_pool = deque(copy.copy(self.specs_to_add)) + while specs_pool: + spec = specs_pool.popleft() + freezes = explicit_spec_package_pool.get(spec.name, set()) + matches = set(ssc.r.find_matches(spec)) + explicit_spec_package_pool[spec.name] = freezes | matches + specs_seen.add(spec) + for prec in matches: + for dep in prec.get('depends', []): + dep = MatchSpec(dep) + if dep not in specs_seen and dep not in specs_pool: + specs_pool.append(dep) + + def pkg_in_shared_pool(prec, shared_pool, known_bad_specs, known_good_specs): + """Determine if a package or its recursive deps conflict with the shared pool""" + if prec.name in shared_pool: + if prec not in shared_pool[prec.name]: + return False + + specs_pool = deque(MatchSpec(_) for _ in prec.depends) + while specs_pool: + spec = specs_pool.popleft() + matches = ssc.r.find_matches(spec) + # match the immediate dependencies + if spec.name in shared_pool: + overlap = shared_pool[spec.name] & set(matches) + if overlap: + # restrict the space further, potentially + shared_pool[spec.name] = overlap + known_good_specs.add(spec) + else: + known_bad_specs.add(spec) + return False + else: + known_good_specs.add(spec) + + # recurse into the next layer of deps + for prec in matches: + for dep in prec.get('depends', []): + dep = MatchSpec(dep) + if not any(dep in col for col in (known_bad_specs, known_good_specs)): + if dep.name in shared_pool: + if (spec.name in shared_pool and not + (shared_pool[spec.name] & set(ssc.r.find_matches(dep)))): + continue + specs_pool.append(dep) + return True + + known_bad_specs = set() + known_good_specs = set() + for pkg_name, spec in iteritems(ssc.specs_map): matches_for_spec = tuple(prec for prec in ssc.solution_precs if spec.match(prec)) if matches_for_spec: @@ -398,11 +459,17 @@ def _add_specs(self, ssc): """) % (pkg_name, spec, dashlist((text_type(s) for s in matches_for_spec), indent=4))) target_prec = matches_for_spec[0] + if ssc.update_modifier == UpdateModifier.FREEZE_INSTALLED: - new_spec = MatchSpec(target_prec) + if pkg_name in context.aggressive_update_packages: + ssc.specs_map[pkg_name] = pkg_name + elif pkg_in_shared_pool(target_prec, explicit_spec_package_pool, + known_bad_specs, known_good_specs): + ssc.specs_map[pkg_name] = target_prec.to_match_spec() + elif pkg_name in ssc.specs_from_history_map: + ssc.specs_map[pkg_name] = ssc.specs_from_history_map[pkg_name] else: - new_spec = MatchSpec(spec, target=target_prec.dist_str()) - ssc.specs_map[pkg_name] = new_spec + ssc.specs_map[pkg_name] = MatchSpec(spec, target=target_prec.dist_str()) log.debug("specs_map with targets: %s", ssc.specs_map) # If we're in UPDATE_ALL mode, we need to drop all the constraints attached to specs, diff --git a/conda/resolve.py b/conda/resolve.py index 491a750eb16..397b159e33e 100644 --- a/conda/resolve.py +++ b/conda/resolve.py @@ -505,11 +505,11 @@ def filter_group(_specs): reduced = nnew < nold if reduced: log.debug('%s: pruned from %d -> %d' % (name, nold, nnew)) - if any(ms.optional for ms in _specs): - return reduced - elif nnew == 0: + if nnew == 0: # Indicates that a conflict was found; we can exit early return None + elif any(ms.optional for ms in _specs): + return reduced # Perform the same filtering steps on any dependencies shared across # *all* packages in the group. Even if just one of the packages does diff --git a/tests/core/test_solve.py b/tests/core/test_solve.py index caa364506b9..8ae699963f4 100644 --- a/tests/core/test_solve.py +++ b/tests/core/test_solve.py @@ -2310,6 +2310,7 @@ def test_freeze_deps_1(): unlink_order = ( 'channel-2::six-1.7.3-py34_0', 'channel-2::python-3.4.5-0', + 'channel-2::xz-5.2.3-0', ) link_order = ( 'channel-2::mkl-2017.0.3-0', diff --git a/tests/test_cli.py b/tests/test_cli.py index 3fb3c7148e1..06378b06ee6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -150,10 +150,10 @@ def test_search_3(self): with make_temp_env() as prefix: stdout, stderr, _ = run_command(Commands.SEARCH, prefix, "*/linux-64::nose==1.3.7[build=py37_2]", "--info", use_exception_handler=True) result = stdout.replace("Loading channels: ...working... done", "") - assert "file name : nose-1.3.7-py37_2.tar.bz2" in result + assert "file name : nose-1.3.7-py37_2.conda" in result assert "name : nose" in result - assert "url : https://repo.anaconda.com/pkgs/main/linux-64/nose-1.3.7-py37_2.tar.bz2" in result - assert "md5 : ff390a1e44d77e54914ca1a2c9e75445" in result + assert "url : https://repo.anaconda.com/pkgs/main/linux-64/nose-1.3.7-py37_2.conda" in result + # assert "md5 : ff390a1e44d77e54914ca1a2c9e75445" in result @pytest.mark.integration def test_search_4(self): diff --git a/tests/test_create.py b/tests/test_create.py index 9dc51ff2b97..1e7d7c30749 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -983,7 +983,8 @@ def test_tarball_install_and_bad_metadata(self): flask_fname = flask_data['fn'] tar_old_path = join(PackageCacheData.first_writable().pkgs_dir, flask_fname) - assert isfile(tar_old_path) + # if a .tar.bz2 is already in the file cache, it's fine. Accept it or the .conda file here. + assert isfile(tar_old_path) or isfile(tar_old_path.replace('.conda', '.tar.bz2')) with pytest.raises(DryRunExit): run_command(Commands.INSTALL, prefix, tar_old_path, "--dry-run") @@ -1036,6 +1037,7 @@ def test_update_with_pinned_packages(self): # regression test for #6914 with make_temp_env("-c", "https://repo.anaconda.com/pkgs/free", "python=2.7.12") as prefix: assert package_is_installed(prefix, "readline=6.2") + # removing the history allows python to be updated too open(join(prefix, 'conda-meta', 'history'), 'w').close() PrefixData._cache_.clear() run_command(Commands.UPDATE, prefix, "readline") @@ -1122,13 +1124,8 @@ def test_install_prune_flag(self): assert package_is_installed(prefix, 'python=3') run_command(Commands.REMOVE, prefix, "flask") assert not package_is_installed(prefix, 'flask') - assert package_is_installed(prefix, 'itsdangerous') - assert package_is_installed(prefix, 'python=3') - - run_command(Commands.INSTALL, prefix, 'pytz', '--prune') - + # this should get pruned when flask is removed assert not package_is_installed(prefix, 'itsdangerous') - assert package_is_installed(prefix, 'pytz') assert package_is_installed(prefix, 'python=3') @pytest.mark.skipif(on_win, reason="readline is only a python dependency on unix") @@ -1220,27 +1217,19 @@ def test_install_features(self): assert package_is_installed(prefix, "numpy") assert package_is_installed(prefix, "nomkl") assert not package_is_installed(prefix, "mkl") - numpy_prec = PrefixData(prefix).get("numpy") - assert "nomkl" in numpy_prec.build with make_temp_env("python=2", "numpy=1.13") as prefix: assert package_is_installed(prefix, "numpy") assert not package_is_installed(prefix, "nomkl") assert package_is_installed(prefix, "mkl") - numpy_prec = PrefixData(prefix).get("numpy") - assert "nomkl" not in numpy_prec.build - run_command(Commands.INSTALL, prefix, "nomkl") + run_command(Commands.INSTALL, prefix, "nomkl", no_capture=True) assert package_is_installed(prefix, "numpy") assert package_is_installed(prefix, "nomkl") - assert package_is_installed(prefix, "mkl") # it's fine for mkl to still be here I guess - numpy_prec = PrefixData(prefix).get("numpy") - assert "nomkl" in numpy_prec.build - - run_command(Commands.INSTALL, prefix, "nomkl", "--prune") - assert not package_is_installed(prefix, "mkl") + assert package_is_installed(prefix, "blas=1.0=openblas") assert not package_is_installed(prefix, "mkl_fft") assert not package_is_installed(prefix, "mkl_random") + assert not package_is_installed(prefix, "mkl") # pruned as an indirect dep def test_clone_offline_simple(self): with make_temp_env("bzip2") as prefix: @@ -1690,9 +1679,11 @@ def test_conda_pip_interop_dependency_satisfied_by_pip(self): assert not error output, _, _ = run_command(Commands.INSTALL, prefix, 'flask', '--dry-run', '--json', - use_exception_handler=True) + use_exception_handler=True) json_obj = json.loads(output) print(json_obj) + # itsdangerous shouldn't be in this list, because it's already present and satisfied + # by the pip package assert any(rec["name"] == "flask" for rec in json_obj["actions"]["LINK"]) assert not any(rec["name"] == "itsdangerous" for rec in json_obj["actions"]["LINK"]) @@ -1848,7 +1839,8 @@ def test_conda_pip_interop_pip_clobbers_conda(self): assert not stderr assert "All requested packages already installed." in stdout - stdout, stderr, _ = run_command(Commands.INSTALL, prefix, "six") + stdout, stderr, _ = run_command(Commands.INSTALL, prefix, "six", "--repodata-fn", + "repodata.json", no_capture=True) assert not stderr assert package_is_installed(prefix, "six>=1.11") output, err, _ = run_command(Commands.RUN, prefix, "python", "-m", "pip", "freeze") @@ -1982,7 +1974,7 @@ def test_conda_pip_interop_compatible_release_operator(self): # Regression test for #7776 # important to start the env with six 1.9. That version forces an upgrade later in the test with make_temp_env("-c", "https://repo.anaconda.com/pkgs/free", "pip=10", "six=1.9", "appdirs", - use_restricted_unicode=on_win) as prefix: + use_restricted_unicode=on_win, no_capture=True) as prefix: run_command(Commands.CONFIG, prefix, "--set", "pip_interop_enabled", "true") assert package_is_installed(prefix, "python") assert package_is_installed(prefix, "six=1.9") @@ -2014,14 +2006,14 @@ def test_conda_pip_interop_compatible_release_operator(self): with pytest.raises(DryRunExit): run_command(Commands.INSTALL, prefix, "-c", "https://repo.anaconda.com/pkgs/free", - "agate=1.6", "--dry-run") + "agate=1.6", "--dry-run", no_capture=True) def test_install_freezes_env_by_default(self): """We pass --no-update-deps/--freeze-installed by default, effectively. This helps speed things up by not considering changes to existing stuff unless the solve ends up unsatisfiable.""" # create an initial env - with make_temp_env("python=2", use_restricted_unicode=on_win) as prefix: + with make_temp_env("python=2", use_restricted_unicode=on_win, no_capture=True) as prefix: assert package_is_installed(prefix, "python=2.7.*") stdout, stderr, _ = run_command(Commands.LIST, prefix, '--json') pkgs = json.loads(stdout) @@ -2030,8 +2022,10 @@ def test_install_freezes_env_by_default(self): if entry['name'] in DEFAULT_AGGRESSIVE_UPDATE_PACKAGES: specs.append(entry['name']) else: - specs.append(entry['channel'] + '/' + entry['platform'] + '::' + - entry['name'] + '=' + entry['version'] + '=' + entry['build_string']) + # python is the only explicit dep. It's the only thing that enters in here. + if entry['name'] == 'python': + specs.append(entry['channel'] + '/' + entry['platform'] + '::' + + entry['name'] + '=' + entry['version'] + '=' + entry['build_string']) specs.append('imagesize') specs = {MatchSpec(s) for s in specs} @@ -2039,11 +2033,11 @@ def test_install_freezes_env_by_default(self): r = conda.core.solve.Resolve(get_index()) reduced_index = r.get_reduced_index([MatchSpec('imagesize')]) - # now add requests to that env. The call to get_reduced_index should include our exact specs + # now add imagesize to that env. The call to get_reduced_index should include our exact specs # for the existing env. doing so will greatly reduce the search space for the initial solve with patch.object(conda.core.solve.Resolve, 'get_reduced_index', return_value=reduced_index) as mock_method: - run_command(Commands.INSTALL, prefix, "imagesize") + run_command(Commands.INSTALL, prefix, "imagesize", no_capture=True) # TODO: this should match the specs above. It does, at least as far as I can tell from text # comparison. Unfortunately, it doesn't evaluate as a match, even though the text all matches. # I suspect some strange equality of objects issue.