Merge pull request conda#3927 from kalefranz/pr-3880

Populate PackageInfo model with info from paths.json

conda/core/install.py

@@ -26,6 +26,7 @@
 from ..gateways.disk.read import collect_all_info_for_package, yield_lines
 from ..gateways.disk.update import _PaddingError, update_prefix
 from ..models.record import Link
+from ..models.package_info import PathType
 from ..utils import on_win
 
 try:
@@ -96,21 +97,22 @@ def _make_link_operations(self, requested_link_type):
         # no side effects in this method!
         package_info = self.package_info
 
-        def make_link_operation(source_short_path):
-            if source_short_path in package_info.has_prefix_files:
+        def make_link_operation(source_path_info):
+            if source_path_info.prefix_placeholder:
                 link_type = LinkType.copy
-                prefix_placehoder, file_mode = package_info.has_prefix_files[source_short_path]
-            elif source_short_path in concatv(package_info.no_link, package_info.soft_links):
+                prefix_placehoder = source_path_info.prefix_placeholder
+                file_mode = source_path_info.file_mode
+            elif source_path_info.no_link or source_path_info.path_type == PathType.softlink:
                 link_type = LinkType.copy
                 prefix_placehoder, file_mode = '', None
             else:
                 link_type = requested_link_type
                 prefix_placehoder, file_mode = '', None
-            is_menu_file = bool(MENU_RE.match(source_short_path))
-            dest_short_path = source_short_path
-            return LinkOperation(source_short_path, dest_short_path, link_type, prefix_placehoder,
-                                 file_mode, is_menu_file)
-        return tuple(make_link_operation(p) for p in package_info.files)
+            is_menu_file = bool(MENU_RE.match(source_path_info.path))
+            dest_short_path = source_path_info.path
+            return LinkOperation(source_path_info.path, dest_short_path, link_type,
+                                 prefix_placehoder, file_mode, is_menu_file)
+        return tuple(make_link_operation(p) for p in package_info.paths)
 
     def _execute_link_operations(self, leaf_directories, link_operations):
         # major side-effects in this method
@@ -186,21 +188,23 @@ def _make_link_operations(self, requested_link_type):
         site_packages_dir = get_site_packages_dir(self.prefix)
         bin_dir = get_bin_directory_short_path()
 
-        def make_link_operation(source_short_path):
+        def make_link_operation(source_path_info):
             # no side effects in this method!
 
             # first part, same as parent class
-            if source_short_path in package_info.has_prefix_files:
+            if source_path_info.prefix_placeholder:
                 link_type = LinkType.copy
-                prefix_placehoder, file_mode = package_info.has_prefix_files[source_short_path]
-            elif source_short_path in concatv(package_info.no_link, package_info.soft_links):
+                prefix_placehoder = source_path_info.prefix_placeholder
+                file_mode = source_path_info.file_mode
+            elif source_path_info.no_link or source_path_info.path_type == PathType.softlink:
                 link_type = LinkType.copy
                 prefix_placehoder, file_mode = '', None
             else:
                 link_type = requested_link_type
                 prefix_placehoder, file_mode = '', None
-            is_menu_file = bool(MENU_RE.match(source_short_path))
+            is_menu_file = bool(MENU_RE.match(source_path_info.path))
 
+            source_short_path = source_path_info.path
             # second part, noarch python-specific
             if source_short_path.startswith('site-packages/'):
                 dest_short_path = site_packages_dir + source_short_path.replace(
@@ -212,7 +216,7 @@ def make_link_operation(source_short_path):
             return LinkOperation(source_short_path, dest_short_path, link_type, prefix_placehoder,
                                  file_mode, is_menu_file)
 
-        return tuple(make_link_operation(p) for p in package_info.files)
+        return tuple(make_link_operation(p) for p in package_info.paths)
 
     def _execute_link_operations(self, leaf_directories, link_operations):
         dest_short_paths = super(NoarchPythonPackageInstaller, self)._execute_link_operations(
@@ -224,7 +228,7 @@ def _execute_link_operations(self, leaf_directories, link_operations):
                                               tuple(op.dest_short_path for op in link_operations))
 
         # create entry points
-        entry_points = self.package_info.noarch.get('entry_points', ())
+        entry_points = self.package_info.noarch.entry_points
         entry_point_paths = []
         for entry_point in entry_points:
             entry_point_paths.extend(create_entry_point(entry_point, self.prefix))

conda/exceptions.py

@@ -404,6 +404,12 @@ def __init__(self, message):
         super(CondaCorruptEnvironmentError, self).__init__(msg)
 
 
+class CondaUpgradeError(CondaError):
+    def __init__(self, message):
+        msg = "Conda upgrade error: %s" % message
+        super(CondaUpgradeError, self).__init__(msg)
+
+
 def print_conda_exception(exception):
     from conda.base.context import context
 

conda/exports.py

@@ -77,7 +77,11 @@
 from conda.base.context import get_prefix as context_get_prefix, non_x86_linux_machines  # NOQA
 non_x86_linux_machines = non_x86_linux_machines
 
-from conda.base.constants import DEFAULT_CHANNELS  # NOQA
+from conda.base.constants import DEFAULT_CHANNELS       # NOQA
+from .models.package_info import PathType               # NOQA
+from ._vendor.auxlib.entity import EntityEncoder        # NOQA
+PathType = PathType
+EntityEncoder = EntityEncoder
 get_prefix = partial(context_get_prefix, conda.base.context.context)
 get_default_urls = lambda: DEFAULT_CHANNELS
 

conda/gateways/disk/read.py

@@ -11,8 +11,9 @@
 from os.path import isfile, islink, join
 
 from ...base.constants import FileMode, PREFIX_PLACEHOLDER, UTF8
-from ...models.package_info import PackageInfoContents
+from ...models.package_info import PathType, PackageInfo, PathInfoV1, PathInfo
 from ...models.record import Record
+from ...exceptions import CondaUpgradeError
 
 log = getLogger(__name__)
 
@@ -45,17 +46,48 @@ def collect_all_info_for_package(extracted_package_directory):
     info_dir = join(extracted_package_directory, 'info')
 
     files_path = join(extracted_package_directory, 'info', 'files')
-    files = tuple(ln for ln in (line.strip() for line in yield_lines(files_path)) if ln)
+    file_json_path = join(extracted_package_directory, 'info', 'files.json')
+
+    if isfile(file_json_path):
+        with open(file_json_path) as file_json:
+            data = json.load(file_json)
+        if data.get('paths_version') != 1:
+            raise CondaUpgradeError("""The current version of conda is too old to install this
+package. (This version only supports files.json schema version 1.)  Please update conda to install
+this package.""")
+
+        paths = (PathInfoV1(**f) for f in data['paths'])
+        index_json_record = read_index_json(extracted_package_directory)
+        noarch = read_noarch(extracted_package_directory)
+        icondata = read_icondata(extracted_package_directory)
+        return PackageInfo(paths=paths, index_json_record=index_json_record, noarch=noarch,
+                           icondata=icondata, paths_version=1)
+    else:
+        files = tuple(ln for ln in (line.strip() for line in yield_lines(files_path)) if ln)
+
+        has_prefix_files = read_has_prefix(join(info_dir, 'has_prefix'))
+        no_link = read_no_link(info_dir)
 
-    has_prefix_files = read_has_prefix(join(info_dir, 'has_prefix'))
-    no_link = read_no_link(info_dir)
-    soft_links = read_soft_links(extracted_package_directory, files)
-    index_json_record = read_index_json(extracted_package_directory)
-    icondata = read_icondata(extracted_package_directory)
-    noarch = read_noarch(extracted_package_directory)
+        path_info_files = []
+        for f in files:
+            path_info = {"_path": f}
+            if f in has_prefix_files.keys():
+                path_info["prefix_placeholder"] = has_prefix_files[f][0]
+                path_info["file_mode"] = has_prefix_files[f][1]
+            if f in no_link:
+                path_info["no_link"] = True
+            if islink(join(extracted_package_directory, f)):
+                path_info["path_type"] = PathType.softlink
+            else:
+                path_info["path_type"] = PathType.hardlink
+            path_info_files.append(PathInfo(**path_info))
 
-    return PackageInfoContents(files, has_prefix_files, no_link, soft_links,
-                               index_json_record, icondata, noarch)
+        index_json_record = read_index_json(extracted_package_directory)
+        icondata = read_icondata(extracted_package_directory)
+        noarch = read_noarch(extracted_package_directory)
+
+        return PackageInfo(paths=path_info_files, index_json_record=index_json_record,
+                           icondata=icondata, noarch=noarch, paths_version=0)
 
 
 def read_noarch(extracted_package_directory):
@@ -64,7 +96,7 @@ def read_noarch(extracted_package_directory):
         with open(noarch_path, 'r') as f:
             return json.loads(f.read())
     else:
-        return {}
+        return None
 
 
 def read_index_json(extracted_package_directory):

conda/models/package_info.py

@@ -1,21 +1,60 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
-from collections import namedtuple
+from enum import Enum
 from logging import getLogger
 
-from .._vendor.auxlib.entity import Entity, ListField
+from .record import Record
+from .._vendor.auxlib.entity import (BooleanField, ComposableField, Entity, EnumField,
+                                     IntegerField, ListField, StringField)
+from ..base.constants import FileMode
 from ..common.compat import string_types
 
 log = getLogger(__name__)
 
 
-PackageInfoContents = namedtuple('PackageInfoContents',
-                                 ('files', 'has_prefix_files', 'no_link', 'soft_links',
-                                  'index_json_record', 'icondata', 'noarch'))
+class NoarchInfo(Entity):
+    type = StringField()
+    entry_points = ListField(string_types, required=False)
 
 
-class PackageInfo(Entity):
+class PathType(Enum):
+    """
+    Refers to if the file in question is hard linked or soft linked. Originally designed to be used
+    in files.json
+    """
+    hardlink = 'hardlink'
+    softlink = 'softlink'
+
+    def __int__(self):
+        return self.value
+
+    def __str__(self):
+        return self.name
+
+
+class PathInfo(Entity):
+    _path = StringField()
+    prefix_placeholder = StringField(required=False, nullable=True)
+    file_mode = EnumField(FileMode, required=False, nullable=True)
+    no_link = BooleanField(required=False, nullable=True)
+    path_type = EnumField(PathType)
 
-    file = ListField(string_types)
-    # TODO: finish this
+    @property
+    def path(self):
+        # because I don't have aliases as an option for entity fields yet
+        return self._path
+
+
+class PathInfoV1(PathInfo):
+    sha256 = StringField()
+    size_in_bytes = IntegerField()
+    inode_paths = ListField(string_types, required=False, nullable=True)
+
+
+class PackageInfo(Entity):
+    paths_version = IntegerField()
+    paths = ListField(PathInfo)
+    index_json_record = ComposableField(Record)
+    icondata = StringField(required=False, nullable=True)
+    noarch = ComposableField(NoarchInfo, required=False, nullable=True)

tests/core/test_install.py

@@ -7,8 +7,8 @@
 from conda.core.install import (PackageInstaller, PackageUninstaller, NoarchPythonPackageInstaller,
                                 LinkOperation)
 from conda.models.dist import Dist
-from conda.models.package_info import PackageInfoContents
-from conda.models.record import Link
+from conda.models.package_info import PathInfo, PackageInfo, NoarchInfo, PathType
+from conda.models.record import Link, Record
 from conda.utils import on_win
 
 try:
@@ -20,15 +20,16 @@
 class TestPackageInstaller(unittest.TestCase):
     def setUp(self):
         self.dist = Dist("channel", "dist_name")
-        files = tuple(["test/path/1", "test/path/2", "test/path/3", "menu/test.json"])
-        has_prefix_files = {"test/path/1": ("/opt/anaconda1anaconda2anaconda3", FileMode.text)}
-        no_link = set(["test/path/2"])
-        soft_links = tuple(["test/path/3"])
-        index_json_records = {"key": "value"}
+        index_json_records = Record(build=0, build_number=0, name="test_foo", version=0)
         icondata = "icondata"
-        noarch = None
-        self.package_info = PackageInfoContents(files, has_prefix_files, no_link, soft_links,
-                                                index_json_records, icondata, noarch)
+        paths = [PathInfo(_path="test/path/1", file_mode=FileMode.text, path_type=PathType.hardlink,
+                          prefix_placeholder="/opt/anaconda1anaconda2anaconda3", ),
+                 PathInfo(_path="test/path/2", no_link=True, path_type=PathType.hardlink),
+                 PathInfo(_path="test/path/3", path_type=PathType.softlink),
+                 PathInfo(_path="menu/test.json", path_type=PathType.hardlink)]
+
+        self.package_info = PackageInfo(paths_version=0, paths=paths, icondata=icondata,
+                                        index_json_record=index_json_records)
 
     def test_make_link_operation(self):
         package_installer = PackageInstaller("prefix", {}, self.dist)
@@ -54,30 +55,28 @@ def test_create_meta(self):
 
         output = package_installer._create_meta(dest_short_paths, LinkType.directory,
                                                 "http://test.url")
-        expected_output = {"key": "value",
-                           "icon": "icon",
-                           "url": "http://test.url",
-                           "files": dest_short_paths,
-                           "link": Link(source="extracted_package_dir", type=LinkType.directory),
-                           "icondata": "icondata"
-                           }
+        expected_output = Record(icon="icon", icondata="icondata", build=0, build_number=0,
+                                 name="test_foo", version=0, url="http://test.url",
+                                 files=dest_short_paths,
+                                 link=Link(source="extracted_package_dir", type=LinkType.directory))
         self.assertEquals(output, expected_output)
 
 
 class TestNoarchPackageInstaller(unittest.TestCase):
     def setUp(self):
         self.dist = Dist("channel", "dist_name")
-        files = tuple(["site-packages/test/1", "python-scripts/test/2", "test/path/3",
-                       "menu/test.json"])
-        has_prefix_files = {"site-packages/test/1": ("/opt/anaconda1anaconda2anaconda3",
-                                                     FileMode.text)}
-        no_link = set(["python-scripts/test/2"])
-        soft_links = tuple(["test/path/3"])
-        index_json_records = {"key": "value"}
+        index_json_records = Record(build=0, build_number=0, name="test_foo", version=0)
         icondata = "icondata"
-        noarch = None
-        self.package_info = PackageInfoContents(files, has_prefix_files, no_link, soft_links,
-                                                index_json_records, icondata, noarch)
+
+        paths = [PathInfo(_path="site-packages/test/1", file_mode=FileMode.text,
+                          path_type=PathType.hardlink,
+                          prefix_placeholder="/opt/anaconda1anaconda2anaconda3", ),
+                 PathInfo(_path="python-scripts/test/2", no_link=True, path_type=PathType.hardlink),
+                 PathInfo(_path="test/path/3", path_type=PathType.softlink),
+                 PathInfo(_path="menu/test.json", path_type=PathType.hardlink)]
+
+        self.package_info = PackageInfo(paths_version=0, paths=paths, icondata=icondata,
+                                        index_json_record=index_json_records)
 
     @patch("conda.core.linked_data.get_python_version_for_prefix", return_value="2.4")
     def test_make_link_operation(self, get_site_packages_dir):
@@ -102,7 +101,6 @@ def test_make_link_operation(self, get_site_packages_dir):
                                  LinkOperation("menu/test.json",
                                                "menu/test.json",
                                                LinkType.softlink, "", None, True)])
-        # import pdb; pdb.set_trace()
         assert output == expected_output
 
 

tests/models/test_package_info.py

@@ -0,0 +1,23 @@
+from conda.models.package_info import PackageInfo, PathInfo, PathType, NoarchInfo
+from conda.base.constants import FileMode
+from conda.models.record import Record
+from unittest import TestCase
+
+
+class DefaultPackageInfo(TestCase):
+    def test_package_info(self):
+        index_json_records = Record(build=0, build_number=0, name="test_foo", version=0)
+        icondata = "icondata"
+        noarch = NoarchInfo(type="python", entry_points=["test:foo"])
+        paths = [PathInfo(_path="test/path/1", file_mode=FileMode.text, path_type=PathType.hardlink,
+                          prefix_placeholder="/opt/anaconda1anaconda2anaconda3", ),
+                 PathInfo(_path="test/path/2", no_link=True, path_type=PathType.hardlink),
+                 PathInfo(_path="test/path/3", path_type=PathType.softlink),
+                 PathInfo(_path="menu/test.json", path_type=PathType.hardlink)]
+
+        package_info = PackageInfo(paths_version=0, paths=paths, icondata=icondata,
+                                   index_json_record=index_json_records, noarch=noarch)
+        self.assertIsInstance(package_info.paths[0], PathInfo)
+        self.assertIsInstance(package_info.index_json_record, Record)
+        self.assertIsInstance(package_info.noarch, NoarchInfo)
+        self.assertEquals(package_info.paths[0].path, "test/path/1")