implement tests for preferring existing pkg cache when using .conda f…

…iles

conda/base/context.py

@@ -158,6 +158,8 @@ class Context(Configuration):
     track_features = SequenceParameter(string_types)
     use_index_cache = PrimitiveParameter(False)
 
+    separate_format_cache = PrimitiveParameter(False)
+
     _root_prefix = PrimitiveParameter("", aliases=('root_dir', 'root_prefix'))
     _envs_dirs = SequenceParameter(string_types, aliases=('envs_dirs', 'envs_path'),
                                    string_delimiter=os.pathsep,
@@ -757,6 +759,7 @@ def category_map(self):
             'extra_safety_checks',
             'shortcuts',
             'non_admin_enabled',
+            'separate_format_cache'
         )),
         ('Conda-build Configuration', (
             'bld_path',
@@ -1080,6 +1083,14 @@ def description_map(self):
                 Enforce available safety guarantees during package installation.
                 The value must be one of 'enabled', 'warn', or 'disabled'.
                 """),
+            'separate_format_cache': dals("""
+                Treat .tar.bz2 files as different from .conda packages when
+                filenames are otherwise similar. This defaults to False, so
+                that your package cache doesn't churn when rolling out the new
+                package format. If you'd rather not assume that a .tar.bz2 and
+                .conda from the same place represent the same content, set this
+                to True.
+            """),
             'extra_safety_checks': dals("""
                 Spend extra time validating package contents.  Currently, runs sha256 verification
                 on every file within each package during installation.

conda/core/package_cache_data.py

@@ -471,8 +471,7 @@ def make_actions_for_record(pref_or_spec):
         #   (1) already extracted, and
         #   (2) matches the md5
         # If one exists, no actions are needed.
-        md5 = pref_or_spec.get('md5')
-        if md5:
+        if pref_or_spec.get('md5'):
             extracted_pcrec = next((
                 pcrec for pcrec in concat(PackageCacheData(pkgs_dir).query(pref_or_spec)
                                           for pkgs_dir in context.pkgs_dirs)
@@ -522,7 +521,8 @@ def make_actions_for_record(pref_or_spec):
                 url=path_to_url(pcrec_from_read_only_cache.package_tarball_full_path),
                 target_pkgs_dir=first_writable_cache.pkgs_dir,
                 target_package_basename=pcrec_from_read_only_cache.fn,
-                md5sum=md5,
+                md5sum=pref_or_spec.get('md5'),
+                sha256sum=pref_or_spec.get('sha256'),
                 expected_size_in_bytes=expected_size_in_bytes,
             )
             trgt_extracted_dirname = strip_pkg_extension(pcrec_from_read_only_cache.fn)[0]
@@ -548,15 +548,16 @@ def make_actions_for_record(pref_or_spec):
             url=url,
             target_pkgs_dir=first_writable_cache.pkgs_dir,
             target_package_basename=pref_or_spec.fn,
-            md5sum=md5,
+            md5sum=pref_or_spec.get('md5'),
+            sha256sum=pref_or_spec.get('sha256'),
             expected_size_in_bytes=expected_size_in_bytes,
         )
         extract_axn = ExtractPackageAction(
             source_full_path=cache_axn.target_full_path,
             target_pkgs_dir=first_writable_cache.pkgs_dir,
             target_extracted_dirname=strip_pkg_extension(pref_or_spec.fn)[0],
             record_or_spec=pref_or_spec,
-            md5sum=md5,
+            md5sum=pref_or_spec.get('md5'),
         )
         return cache_axn, extract_axn
 

conda/core/path_actions.py

@@ -1066,11 +1066,12 @@ def target_full_path(self):
 class CacheUrlAction(PathAction):
 
     def __init__(self, url, target_pkgs_dir, target_package_basename,
-                 md5sum=None, expected_size_in_bytes=None):
+                 md5sum=None, sha256sum=None, expected_size_in_bytes=None):
         self.url = url
         self.target_pkgs_dir = target_pkgs_dir
         self.target_package_basename = target_package_basename
         self.md5sum = md5sum
+        self.sha256sum = sha256sum
         self.expected_size_in_bytes = expected_size_in_bytes
         self.hold_path = self.target_full_path + CONDA_TEMP_EXTENSION
 
@@ -1144,7 +1145,7 @@ def execute(self, progress_update_callback=None):
                     target_package_cache._urls_data.add_url(self.url)
 
         else:
-            download(self.url, self.target_full_path, md5=self.md5sum,
+            download(self.url, self.target_full_path, md5=self.md5sum, sha256=self.sha256sum,
                      progress_update_callback=progress_update_callback)
             target_package_cache._urls_data.add_url(self.url)
 

conda/models/records.py

@@ -255,12 +255,15 @@ def _pkey(self):
         try:
             return self.__pkey
         except AttributeError:
-            # NOTE: fn is included to distinguish between .conda and .tar.bz2 packages
-            __pkey = self.__pkey = (
+            __pkey = self.__pkey = [
                 self.channel.canonical_name, self.subdir, self.name,
-                self.version, self.build_number, self.build, self.fn
-            )
-            return __pkey
+                self.version, self.build_number, self.build
+            ]
+            # NOTE: fn is included to distinguish between .conda and .tar.bz2 packages
+            if context.separate_format_cache:
+                __pkey.append(self.fn)
+            self.__pkey = tuple(__pkey)
+            return self.__pkey
 
     def __hash__(self):
         try:

tests/core/test_package_cache_data.py

@@ -2,6 +2,8 @@
 from os.path import abspath, basename, dirname, join
 
 from conda.base.constants import PACKAGE_CACHE_MAGIC_FILE
+from conda.base.context import conda_tests_ctxt_mgmt_def_pol
+from conda.common.io import env_vars
 from conda.core.index import get_index
 from conda.core.package_cache_data import PackageCacheData, ProgressiveFetchExtract
 from conda.core.path_actions import CacheUrlAction
@@ -66,10 +68,10 @@ def test_ProgressiveFetchExtract_prefers_conda_v2_format():
     assert extract_action.source_full_path.endswith('.conda')
 
 
-def test_tar_bz2_in_pkg_cache_doesnt_overwrite_conda_pkg():
+def test_tar_bz2_in_pkg_cache_used_instead_of_conda_pkg():
     """
     Test that if a .tar.bz2 package is downloaded and extracted in a package cache, the
-    complementary .conda package replaces it if that's what is requested.
+    complementary .conda package is not downloaded/extracted
     """
     with make_temp_package_cache() as pkgs_dir:
         # Cache the .tar.bz2 file in the package cache and extract it
@@ -95,87 +97,130 @@ def test_tar_bz2_in_pkg_cache_doesnt_overwrite_conda_pkg():
         assert len(pfe.cache_actions) == 0
         assert len(pfe.extract_actions) == 0
 
-        # Now ensure download/extract for the complementary .conda package replaces the
-        # extracted .tar.bz2
+        # Now ensure download/extract for the complementary .conda package uses the cache
         pfe = ProgressiveFetchExtract((zlib_conda_prec,))
         pfe.prepare()
-        assert len(pfe.cache_actions) == 1
-        assert len(pfe.extract_actions) == 1
-        cache_action = pfe.cache_actions[0]
-        extact_action = pfe.extract_actions[0]
-        assert basename(cache_action.target_full_path) == zlib_conda_fn
-        assert cache_action.target_full_path == extact_action.source_full_path
-        assert basename(extact_action.target_full_path) == zlib_base_fn
-
-        pfe.execute()
-
-        with open(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh:
-            repodata_record = json.load(fh)
-        assert repodata_record["fn"] == zlib_conda_fn
+        assert len(pfe.cache_actions) == 0
+        assert len(pfe.extract_actions) == 0
 
         # Now check urls.txt to make sure extensions are included.
         urls_text = tuple(yield_lines(join(pkgs_dir, "urls.txt")))
         assert urls_text[0] == zlib_tar_bz2_prec.url
-        assert urls_text[1] == zlib_conda_prec.url
+
+
+def test_tar_bz2_in_pkg_cache_doesnt_overwrite_conda_pkg():
+    """
+    Test that if a .tar.bz2 package is downloaded and extracted in a package cache, the
+    complementary .conda package replaces it if that's what is requested.
+    """
+    with env_vars({'CONDA_SEPARATE_FORMAT_CACHE': True},
+                  stack_callback=conda_tests_ctxt_mgmt_def_pol):
+        with make_temp_package_cache() as pkgs_dir:
+            # Cache the .tar.bz2 file in the package cache and extract it
+            pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 1
+            assert len(pfe.extract_actions) == 1
+            cache_action = pfe.cache_actions[0]
+            extact_action = pfe.extract_actions[0]
+            assert basename(cache_action.target_full_path) == zlib_tar_bz2_fn
+            assert cache_action.target_full_path == extact_action.source_full_path
+            assert basename(extact_action.target_full_path) == zlib_base_fn
+
+            # Go ahead with executing download and extract now
+            pfe.execute()
+
+            assert isfile(join(pkgs_dir, zlib_tar_bz2_fn))
+            assert isfile(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json"))
+
+            # Ensure second download/extract is a no-op
+            pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 0
+            assert len(pfe.extract_actions) == 0
+
+            # Now ensure download/extract for the complementary .conda package replaces the
+            # extracted .tar.bz2
+            pfe = ProgressiveFetchExtract((zlib_conda_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 1
+            assert len(pfe.extract_actions) == 1
+            cache_action = pfe.cache_actions[0]
+            extact_action = pfe.extract_actions[0]
+            assert basename(cache_action.target_full_path) == zlib_conda_fn
+            assert cache_action.target_full_path == extact_action.source_full_path
+            assert basename(extact_action.target_full_path) == zlib_base_fn
+
+            pfe.execute()
+
+            with open(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh:
+                repodata_record = json.load(fh)
+            assert repodata_record["fn"] == zlib_conda_fn
+
+            # Now check urls.txt to make sure extensions are included.
+            urls_text = tuple(yield_lines(join(pkgs_dir, "urls.txt")))
+            assert urls_text[0] == zlib_tar_bz2_prec.url
+            assert urls_text[1] == zlib_conda_prec.url
 
 
 def test_conda_pkg_in_pkg_cache_doesnt_overwrite_tar_bz2():
     """
     Test that if a .conda package is downloaded and extracted in a package cache, the
     complementary .tar.bz2 package replaces it if that's what is requested.
     """
-    with make_temp_package_cache() as pkgs_dir:
-        # Cache the .conda file in the package cache and extract it
-        pfe = ProgressiveFetchExtract((zlib_conda_prec,))
-        pfe.prepare()
-        assert len(pfe.cache_actions) == 1
-        assert len(pfe.extract_actions) == 1
-        cache_action = pfe.cache_actions[0]
-        extact_action = pfe.extract_actions[0]
-        assert basename(cache_action.target_full_path) == zlib_conda_fn
-        assert cache_action.target_full_path == extact_action.source_full_path
-        assert basename(extact_action.target_full_path) == zlib_base_fn
-
-        # Go ahead with executing download and extract now
-        pfe.execute()
-
-        assert isfile(join(pkgs_dir, zlib_conda_fn))
-        assert isfile(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json"))
-
-        # Ensure second download/extract is a no-op
-        pfe = ProgressiveFetchExtract((zlib_conda_prec,))
-        pfe.prepare()
-        assert len(pfe.cache_actions) == 0
-        assert len(pfe.extract_actions) == 0
-
-        # Now ensure download/extract for the complementary .conda package replaces the
-        # extracted .tar.bz2
-        pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec,))
-        pfe.prepare()
-        assert len(pfe.cache_actions) == 1
-        assert len(pfe.extract_actions) == 1
-        cache_action = pfe.cache_actions[0]
-        extact_action = pfe.extract_actions[0]
-        assert basename(cache_action.target_full_path) == zlib_tar_bz2_fn
-        assert cache_action.target_full_path == extact_action.source_full_path
-        assert basename(extact_action.target_full_path) == zlib_base_fn
-
-        pfe.execute()
-
-        with open(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh:
-            repodata_record = json.load(fh)
-        assert repodata_record["fn"] == zlib_tar_bz2_fn
+    with env_vars({'CONDA_SEPARATE_FORMAT_CACHE': True},
+                  stack_callback=conda_tests_ctxt_mgmt_def_pol):
+        with make_temp_package_cache() as pkgs_dir:
+            # Cache the .conda file in the package cache and extract it
+            pfe = ProgressiveFetchExtract((zlib_conda_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 1
+            assert len(pfe.extract_actions) == 1
+            cache_action = pfe.cache_actions[0]
+            extact_action = pfe.extract_actions[0]
+            assert basename(cache_action.target_full_path) == zlib_conda_fn
+            assert cache_action.target_full_path == extact_action.source_full_path
+            assert basename(extact_action.target_full_path) == zlib_base_fn
+
+            # Go ahead with executing download and extract now
+            pfe.execute()
+
+            assert isfile(join(pkgs_dir, zlib_conda_fn))
+            assert isfile(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json"))
+
+            # Ensure second download/extract is a no-op
+            pfe = ProgressiveFetchExtract((zlib_conda_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 0
+            assert len(pfe.extract_actions) == 0
+
+            # Now ensure download/extract for the complementary .conda package replaces the
+            # extracted .tar.bz2
+            pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec,))
+            pfe.prepare()
+            assert len(pfe.cache_actions) == 1
+            assert len(pfe.extract_actions) == 1
+            cache_action = pfe.cache_actions[0]
+            extact_action = pfe.extract_actions[0]
+            assert basename(cache_action.target_full_path) == zlib_tar_bz2_fn
+            assert cache_action.target_full_path == extact_action.source_full_path
+            assert basename(extact_action.target_full_path) == zlib_base_fn
+
+            pfe.execute()
+
+            with open(join(pkgs_dir, zlib_base_fn, "info", "repodata_record.json")) as fh:
+                repodata_record = json.load(fh)
+            assert repodata_record["fn"] == zlib_tar_bz2_fn
 
 
 def test_tar_bz2_in_cache_not_extracted():
     """
     Test that if a .tar.bz2 exists in the package cache (not extracted), and the complementary
-    .conda package is requested, the .tar.bz2 package is ignored and the .conda package is
-    downloaded and extracted.
+    .conda package is requested, the .tar.bz2 package in the cache is used by default.
     """
     with make_temp_package_cache() as pkgs_dir:
         copy(join(CHANNEL_DIR, subdir, zlib_tar_bz2_fn), join(pkgs_dir, zlib_tar_bz2_fn))
-        pfe = ProgressiveFetchExtract((zlib_conda_prec,))
+        pfe = ProgressiveFetchExtract((zlib_tar_bz2_prec,))
         pfe.prepare()
         assert len(pfe.cache_actions) == 1
         assert len(pfe.extract_actions) == 1
@@ -185,7 +230,13 @@ def test_tar_bz2_in_cache_not_extracted():
         pkgs_dir_files = listdir(pkgs_dir)
         assert zlib_base_fn in pkgs_dir_files
         assert zlib_tar_bz2_fn in pkgs_dir_files
-        assert zlib_conda_fn in pkgs_dir_files
+
+        # Now ensure download/extract for the complementary .conda package uses the
+        # extracted .tar.bz2
+        pfe = ProgressiveFetchExtract((zlib_conda_prec,))
+        pfe.prepare()
+        assert len(pfe.cache_actions) == 0
+        assert len(pfe.extract_actions) == 0
 
 
 def test_instantiating_package_cache_when_both_tar_bz2_and_conda_exist():