separate conda package metadata under new root-level key in repodata (c…

…onda#8639)

separate conda package metadata under new root-level key in repodata

.gitignore

@@ -35,3 +35,4 @@ miniconda*.exe
 .pytest_cache/
 docs/source/_build
 **/.vscode
+conda.tmp/

conda/base/constants.py

@@ -134,7 +134,14 @@
 # Maximum priority, reserved for packages we really want to remove
 MAX_CHANNEL_PRIORITY = 10000
 
-CONDA_TARBALL_EXTENSION = '.tar.bz2'
+CONDA_PACKAGE_EXTENSION_V1 = ".tar.bz2"
+CONDA_PACKAGE_EXTENSION_V2 = ".conda"
+CONDA_PACKAGE_EXTENSIONS = (
+    CONDA_PACKAGE_EXTENSION_V2,
+    CONDA_PACKAGE_EXTENSION_V1,
+)
+CONDA_TARBALL_EXTENSION = CONDA_PACKAGE_EXTENSION_V1  # legacy support for conda-build; remove this line  # NOQA
+CONDA_TEMP_EXTENSION = '.c~'
 
 UNKNOWN_CHANNEL = "<unknown>"
 

conda/base/context.py

@@ -158,6 +158,8 @@ class Context(Configuration):
     track_features = SequenceParameter(string_types)
     use_index_cache = PrimitiveParameter(False)
 
+    separate_format_cache = PrimitiveParameter(False)
+
     _root_prefix = PrimitiveParameter("", aliases=('root_dir', 'root_prefix'))
     _envs_dirs = SequenceParameter(string_types, aliases=('envs_dirs', 'envs_path'),
                                    string_delimiter=os.pathsep,
@@ -757,6 +759,7 @@ def category_map(self):
             'extra_safety_checks',
             'shortcuts',
             'non_admin_enabled',
+            'separate_format_cache'
         )),
         ('Conda-build Configuration', (
             'bld_path',
@@ -1080,6 +1083,14 @@ def description_map(self):
                 Enforce available safety guarantees during package installation.
                 The value must be one of 'enabled', 'warn', or 'disabled'.
                 """),
+            'separate_format_cache': dals("""
+                Treat .tar.bz2 files as different from .conda packages when
+                filenames are otherwise similar. This defaults to False, so
+                that your package cache doesn't churn when rolling out the new
+                package format. If you'd rather not assume that a .tar.bz2 and
+                .conda from the same place represent the same content, set this
+                to True.
+            """),
             'extra_safety_checks': dals("""
                 Spend extra time validating package contents.  Currently, runs sha256 verification
                 on every file within each package during installation.

conda/cli/install.py

@@ -15,7 +15,7 @@
 from ..base.context import context, locate_prefix_by_name
 from ..common.compat import on_win, text_type
 from ..common.constants import NULL
-from ..common.path import paths_equal
+from ..common.path import paths_equal, is_package_file
 from ..core.index import calculate_channel_urls, get_index
 from ..core.prefix_data import PrefixData
 from ..core.solve import DepsModifier, Solver
@@ -191,7 +191,7 @@ def install(args, parser, command='install'):
         'use_local': args.use_local
     }
 
-    num_cp = sum(s.endswith('.tar.bz2') for s in args_packages)
+    num_cp = sum(is_package_file(s) for s in args_packages)
     if num_cp:
         if num_cp == len(args_packages):
             explicit(args_packages, prefix, verbose=not context.quiet)

conda/cli/main_clean.py

@@ -6,12 +6,11 @@
 from collections import defaultdict
 import fnmatch
 from logging import getLogger
-from os import listdir, lstat, walk, unlink
-from os.path import getsize, isdir, join, exists
+from os import listdir, lstat, unlink, walk
+from os.path import exists, getsize, isdir, join
 import sys
 
-from ..base.constants import CONDA_TARBALL_EXTENSION
-from ..common.constants import CONDA_TEMP_EXTENSION
+from ..base.constants import CONDA_PACKAGE_EXTENSIONS, CONDA_TEMP_EXTENSION
 from ..base.context import context
 
 log = getLogger(__name__)
@@ -21,14 +20,14 @@ def find_tarballs():
     from ..core.package_cache_data import PackageCacheData
     pkgs_dirs = defaultdict(list)
     totalsize = 0
-    part_ext = CONDA_TARBALL_EXTENSION + '.part'
+    part_ext = tuple(e + '.part' for e in CONDA_PACKAGE_EXTENSIONS)
     for package_cache in PackageCacheData.writable_caches(context.pkgs_dirs):
         pkgs_dir = package_cache.pkgs_dir
         if not isdir(pkgs_dir):
             continue
         root, _, filenames = next(walk(pkgs_dir))
         for fn in filenames:
-            if fn.endswith(CONDA_TARBALL_EXTENSION) or fn.endswith(part_ext):
+            if fn.endswith(CONDA_PACKAGE_EXTENSIONS) or fn.endswith(part_ext):
                 pkgs_dirs[pkgs_dir].append(fn)
                 totalsize += getsize(join(root, fn))
 

conda/cli/main_package.py

@@ -12,6 +12,7 @@
 import tempfile
 
 from .._vendor.auxlib.entity import EntityEncoder
+from ..base.constants import CONDA_PACKAGE_EXTENSION_V1
 from ..base.context import context
 from ..common.compat import PY3
 from ..common.path import paths_equal
@@ -189,7 +190,7 @@ def make_tarbz2(prefix, name='unknown', version='0.0', build_number=0,
         requires_py = False
 
     info = create_info(name, version, build_number, requires_py)
-    tarbz2_fn = '%(name)s-%(version)s-%(build)s.tar.bz2' % info
+    tarbz2_fn = ('%(name)s-%(version)s-%(build)s' % info) + CONDA_PACKAGE_EXTENSION_V1
     create_conda_pkg(prefix, files, info, tarbz2_fn)
     print('# success')
     print(tarbz2_fn)

conda/common/constants.py

@@ -10,5 +10,3 @@
 #   to null, or the key didn't exist at all.  There could be a bit of potential confusion here,
 #   because in python null == None, while here I'm defining NULL to mean 'not defined'.
 NULL = NULL
-
-CONDA_TEMP_EXTENSION = '.c~'

conda/common/path.py

@@ -310,3 +310,40 @@ def _translation(found_path):  # NOQA
 def which(executable):
     from distutils.spawn import find_executable
     return find_executable(executable)
+
+
+def strip_pkg_extension(path):
+    """
+    Examples:
+        >>> strip_pkg_extension("/path/_license-1.1-py27_1.tar.bz2")
+        ('/path/_license-1.1-py27_1', '.tar.bz2')
+        >>> strip_pkg_extension("/path/_license-1.1-py27_1.conda")
+        ('/path/_license-1.1-py27_1', '.conda')
+        >>> strip_pkg_extension("/path/_license-1.1-py27_1")
+        ('/path/_license-1.1-py27_1', None)
+    """
+    # NOTE: not using CONDA_TARBALL_EXTENSION_V1 or CONDA_TARBALL_EXTENSION_V2 to comply with
+    #       import rules and to avoid a global lookup.
+    if path[-6:] == ".conda":
+        return path[:-6], ".conda"
+    elif path[-8:] == ".tar.bz2":
+        return path[:-8], ".tar.bz2"
+    elif path[-5:] == ".json":
+        return path[:-5], ".json"
+    else:
+        return path, None
+
+
+def is_package_file(path):
+    """
+    Examples:
+        >>> is_package_file("/path/_license-1.1-py27_1.tar.bz2")
+        True
+        >>> is_package_file("/path/_license-1.1-py27_1.conda")
+        True
+        >>> is_package_file("/path/_license-1.1-py27_1")
+        False
+    """
+    # NOTE: not using CONDA_TARBALL_EXTENSION_V1 or CONDA_TARBALL_EXTENSION_V2 to comply with
+    #       import rules and to avoid a global lookup.
+    return path[-6:] == ".conda" or path[-8:] == ".tar.bz2"

conda/common/url.py

@@ -10,7 +10,7 @@
 import socket
 
 from .compat import input, on_win
-from .path import split_filename
+from .path import split_filename, strip_pkg_extension
 from .._vendor.auxlib.decorators import memoize
 from .._vendor.urllib3.exceptions import LocationParseError
 from .._vendor.urllib3.util.url import Url, parse_url
@@ -298,12 +298,6 @@ def has_platform(url, known_subdirs):
     return maybe_a_platform in known_subdirs and maybe_a_platform or None
 
 
-def _split_package_filename(url):
-    cleaned_url, package_filename = (url.rsplit('/', 1) if url.endswith(('.tar.bz2', '.json'))
-                                     else (url, None))
-    return cleaned_url, package_filename
-
-
 def split_scheme_auth_token(url):
     """
     Examples:
@@ -325,7 +319,8 @@ def split_conda_url_easy_parts(url, known_subdirs):
     # scheme, auth, token, platform, package_filename, host, port, path, query
     cleaned_url, token = split_anaconda_token(url)
     cleaned_url, platform = split_platform(cleaned_url, known_subdirs)
-    cleaned_url, package_filename = _split_package_filename(cleaned_url)
+    _, ext = strip_pkg_extension(cleaned_url)
+    cleaned_url, package_filename = cleaned_url.rsplit('/', 1) if ext else (cleaned_url, None)
 
     # TODO: split out namespace using regex
 

conda/core/index.py

@@ -3,9 +3,8 @@
 # SPDX-License-Identifier: BSD-3-Clause
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from logging import getLogger
-
 from itertools import chain
+from logging import getLogger
 
 from .package_cache_data import PackageCacheData
 from .prefix_data import PrefixData
@@ -16,10 +15,11 @@
 from ..common.compat import itervalues
 from ..common.io import ThreadLimitedThreadPoolExecutor, as_completed, dashlist, time_recorder
 from ..exceptions import ChannelNotAllowed, InvalidSpec
+from ..gateways.logging import initialize_logging
 from ..models.channel import Channel, all_channel_urls
-from ..models.match_spec import MatchSpec
-from ..models.records import EMPTY_LINK, PackageCacheRecord, PrefixRecord, PackageRecord
 from ..models.enums import PackageType
+from ..models.match_spec import MatchSpec
+from ..models.records import EMPTY_LINK, PackageCacheRecord, PackageRecord, PrefixRecord
 
 log = getLogger(__name__)
 
@@ -47,6 +47,8 @@ def get_index(channel_urls=(), prepend=True, platform=None,
     If platform=None, then the current platform is used.
     If prefix is supplied, then the packages installed in that prefix are added.
     """
+    initialize_logging()  # needed in case this function is called directly as a public API
+
     if context.offline and unknown is None:
         unknown = True
 

conda/core/link.py

@@ -23,16 +23,16 @@
 from .._vendor.auxlib.collection import first
 from .._vendor.auxlib.ish import dals
 from .._vendor.toolz import concat, concatv, interleave, take
-from ..base.constants import DEFAULTS_CHANNEL_NAME, SafetyChecks, PREFIX_MAGIC_FILE
+from ..base.constants import DEFAULTS_CHANNEL_NAME, PREFIX_MAGIC_FILE, SafetyChecks
 from ..base.context import context
 from ..common.compat import ensure_text_type, iteritems, itervalues, odict, on_win, text_type
 from ..common.io import Spinner, dashlist, time_recorder
 from ..common.path import (explode_directories, get_all_directories, get_major_minor_version,
                            get_python_site_packages_short_path)
 from ..common.signals import signal_handler
-from ..exceptions import (DisallowedPackageError, KnownPackageClobberError, LinkError, RemoveError,
-                          SharedLinkPathClobberError, UnknownPackageClobberError, maybe_raise,
-                          EnvironmentNotWritableError)
+from ..exceptions import (DisallowedPackageError, EnvironmentNotWritableError,
+                          KnownPackageClobberError, LinkError, RemoveError,
+                          SharedLinkPathClobberError, UnknownPackageClobberError, maybe_raise)
 from ..gateways.disk import mkdir_p
 from ..gateways.disk.delete import rm_rf
 from ..gateways.disk.read import isfile, lexists, read_package_info
@@ -41,7 +41,7 @@
 from ..models.enums import LinkType
 from ..models.version import VersionOrder
 from ..resolve import MatchSpec
-from ..utils import human_bytes, wrap_subprocess_call, ensure_comspec_set
+from ..utils import ensure_comspec_set, human_bytes, wrap_subprocess_call
 
 log = getLogger(__name__)
 
@@ -872,7 +872,7 @@ def print_dists(dists_extras):
             total_download_bytes = 0
             for prec in sorted(change_report.fetch_precs,
                                key=lambda x: convert_namekey(x.namekey)):
-                size = prec.size if prec.fn.endswith('.tar.bz2') else prec.conda_size
+                size = prec.size
                 extra = '%15s' % human_bytes(size)
                 total_download_bytes += size
                 schannel = channel_filt(text_type(prec.channel.canonical_name))