MAINT: Upgrade numpy and fix warnings.

Mostly fixes ambiguous calls to numpy.full, and uses explicitly-united
NaT values.

etc/requirements.txt

@@ -24,7 +24,7 @@ six==1.9.0
 # For fetching remote data
 requests==2.9.1
 
-Cython==0.22.1
+Cython==0.23.4
 
 # faster OrderedDict
 cyordereddict==0.2.2
@@ -41,7 +41,7 @@ networkx==1.9.1
 numexpr==2.4.3
 
 # On disk storage format for pipeline data.
-bcolz==0.10.0
+bcolz==0.12.1
 
 # Command line interface helper
 click==4.0.0

setup.py

@@ -24,6 +24,7 @@
     join,
 )
 from distutils.version import StrictVersion
+from pkg_resources import resource_filename
 from setuptools import (
     Extension,
     find_packages,
@@ -33,43 +34,65 @@
 import versioneer
 
 
-class LazyCythonizingList(list):
-    cythonized = False
-
-    def lazy_cythonize(self):
-        if self.cythonized:
-            return
-        self.cythonized = True
-
-        from Cython.Build import cythonize
-        from numpy import get_include
-
-        self[:] = cythonize(
-            [
-                Extension(*ext_args, include_dirs=[get_include()])
-                for ext_args in self
-            ]
+class LazyBuildExtCommandClass(dict):
+    """
+    Lazy command class that defers operations requiring Cython and numpy until
+    they've actually been downloaded and installed by setup_requires.
+    """
+    def __contains__(self, key):
+        return (
+            key == 'build_ext'
+            or super(LazyBuildExtCommandClass, self).__contains__(key)
         )
 
-    def __iter__(self):
-        self.lazy_cythonize()
-        return super(LazyCythonizingList, self).__iter__()
-
-    def __getitem__(self, num):
-        self.lazy_cythonize()
-        return super(LazyCythonizingList, self).__getitem__(num)
-
-
-ext_modules = LazyCythonizingList([
-    ('zipline.assets._assets', ['zipline/assets/_assets.pyx']),
-    ('zipline.lib.adjustment', ['zipline/lib/adjustment.pyx']),
-    ('zipline.lib._float64window', ['zipline/lib/_float64window.pyx']),
-    ('zipline.lib._int64window', ['zipline/lib/_int64window.pyx']),
-    ('zipline.lib._uint8window', ['zipline/lib/_uint8window.pyx']),
-    ('zipline.lib.rank', ['zipline/lib/rank.pyx']),
-    ('zipline.data._equities', ['zipline/data/_equities.pyx']),
-    ('zipline.data._adjustments', ['zipline/data/_adjustments.pyx']),
-])
+    def __setitem__(self, key, value):
+        if key == 'build_ext':
+            raise AssertionError("build_ext overridden!")
+        super(LazyBuildExtCommandClass, self).__setitem__(key, value)
+
+    def __getitem__(self, key):
+        if key != 'build_ext':
+            return super(LazyBuildExtCommandClass, self).__getitem__(key)
+
+        from Cython.Distutils import build_ext as cython_build_ext
+
+        class build_ext(cython_build_ext):
+            """
+            Custom build_ext command that lazily adds numpy's include_dir to
+            extensions.
+            """
+            def build_extensions(self):
+                """
+                Lazily append numpy's include directory to Extension includes.
+
+                This is done here rather than at module scope because setup.py
+                may be run before numpy has been installed, in which case
+                importing numpy and calling `numpy.get_include()` will fail.
+                """
+                numpy_incl = resource_filename('numpy', 'core/include')
+                for ext in self.extensions:
+                    ext.include_dirs.append(numpy_incl)
+
+                # This explicitly calls the superclass method rather than the
+                # usual super() invocation because distutils' build_class, of
+                # which Cython's build_ext is a subclass, is an old-style class
+                # in Python 2, which doesn't support `super`.
+                cython_build_ext.build_extensions(self)
+        return build_ext
+
+
+ext_modules = [
+    Extension('zipline.assets._assets', ['zipline/assets/_assets.pyx']),
+    Extension('zipline.lib.adjustment', ['zipline/lib/adjustment.pyx']),
+    Extension(
+        'zipline.lib._float64window', ['zipline/lib/_float64window.pyx']
+    ),
+    Extension('zipline.lib._int64window', ['zipline/lib/_int64window.pyx']),
+    Extension('zipline.lib._uint8window', ['zipline/lib/_uint8window.pyx']),
+    Extension('zipline.lib.rank', ['zipline/lib/rank.pyx']),
+    Extension('zipline.data._equities', ['zipline/data/_equities.pyx']),
+    Extension('zipline.data._adjustments', ['zipline/data/_adjustments.pyx']),
+]
 
 
 STR_TO_CMP = {
@@ -116,9 +139,8 @@ def _filter_requirements(lines_iter):
             yield requirement
 
 
-REQ_UPPER_BOUNDS = {
-    'numpy': '<1.10',
-}
+# We don't currently have any known upper bounds.
+REQ_UPPER_BOUNDS = {}
 
 
 def _with_bounds(req):
@@ -183,11 +205,12 @@ def extras_requires(conda_format=False):
     }
 
 
-def module_requirements(requirements_path, module_names):
+def module_requirements(requirements_path, module_names, strict_bounds):
     module_names = set(module_names)
     found = set()
     module_lines = []
-    for line in read_requirements(requirements_path, strict_bounds=True):
+    for line in read_requirements(requirements_path,
+                                  strict_bounds=strict_bounds):
         match = REQ_PATTERN.match(line)
         if match is None:
             raise AssertionError("Could not parse requirement: '%s'" % line)
@@ -203,38 +226,12 @@ def module_requirements(requirements_path, module_names):
         )
     return module_lines
 
-
-def pre_setup():
-    if not set(sys.argv) & {'install', 'develop', 'egg_info', 'bdist_wheel'}:
-        return
-
-    try:
-        import pip
-        if StrictVersion(pip.__version__) < StrictVersion('7.1.0'):
-            raise AssertionError(
-                "Zipline installation requires pip>=7.1.0, but your pip "
-                "version is {version}. \n"
-                "You can upgrade your pip with "
-                "'pip install --upgrade pip'.".format(
-                    version=pip.__version__,
-                )
-            )
-    except ImportError:
-        raise AssertionError("Zipline installation requires pip")
-
-    required = ('Cython', 'numpy')
-    for line in module_requirements('etc/requirements.txt', required):
-        pip.main(['install', line])
-
-
-pre_setup()
-
 conda_build = os.path.basename(sys.argv[0]) == 'conda-build'
 
 setup(
     name='zipline',
     version=versioneer.get_version(),
-    cmdclass=versioneer.get_cmdclass(),
+    cmdclass=LazyBuildExtCommandClass(versioneer.get_cmdclass()),
     description='A backtester for financial algorithms.',
     author='Quantopian Inc.',
     author_email='[email protected]',
@@ -258,5 +255,10 @@ def pre_setup():
     ],
     install_requires=install_requires(conda_format=conda_build),
     extras_require=extras_requires(conda_format=conda_build),
+    setup_requires=module_requirements(
+        'etc/requirements.txt',
+        ('Cython', 'numpy'),
+        strict_bounds=False,
+    ),
     url="http://zipline.io",
 )

tests/pipeline/test_earnings.py

@@ -26,7 +26,7 @@
     SID_FIELD_NAME,
     TS_FIELD_NAME,
 )
-from zipline.utils.numpy_utils import make_datetime64D, np_NaT
+from zipline.utils.numpy_utils import make_datetime64D, NaTD
 from zipline.utils.test_utils import (
     make_simple_equity_info,
     tmp_asset_finder,
@@ -234,7 +234,7 @@ def _compute_busday_offsets(announcement_dates):
 
         # Set NaTs to 0 temporarily because busday_count doesn't support NaT.
         # We fill these entries with NaNs later.
-        whereNaT = raw_announce_dates == np_NaT
+        whereNaT = raw_announce_dates == NaTD
         raw_announce_dates[whereNaT] = make_datetime64D(0)
 
         # The abs call here makes it so that we can use this function to

tests/pipeline/test_engine.py

@@ -257,7 +257,7 @@ def test_single_factor(self):
 
             check_arrays(
                 result['f'].unstack().values,
-                full(result_shape, expected_result),
+                full(result_shape, expected_result, dtype=float),
             )
 
     def test_multiple_rolling_factors(self):
@@ -295,16 +295,16 @@ def test_multiple_rolling_factors(self):
         # row-wise sum over an array whose values are all (1 - 2)
         check_arrays(
             results['short'].unstack().values,
-            full(shape, -short_factor.window_length),
+            full(shape, -short_factor.window_length, dtype=float),
         )
         check_arrays(
             results['long'].unstack().values,
-            full(shape, -long_factor.window_length),
+            full(shape, -long_factor.window_length, dtype=float),
         )
         # row-wise sum over an array whose values are all (1 - 3)
         check_arrays(
             results['high'].unstack().values,
-            full(shape, -2 * high_factor.window_length),
+            full(shape, -2 * high_factor.window_length, dtype=float),
         )
 
     def test_numeric_factor(self):
@@ -398,13 +398,19 @@ def test_rolling_and_nonrolling(self):
         result_shape = (len(result_index),)
         check_arrays(
             result['sumdiff'],
-            Series(index=result_index, data=full(result_shape, -3)),
+            Series(
+                index=result_index,
+                data=full(result_shape, -3, dtype=float),
+            ),
         )
 
         for name, const in [('open', 1), ('close', 2), ('volume', 3)]:
             check_arrays(
                 result[name],
-                Series(index=result_index, data=full(result_shape, const)),
+                Series(
+                    index=result_index,
+                    data=full(result_shape, const, dtype=float),
+                ),
             )
 
     def test_loader_given_multiple_columns(self):
@@ -471,19 +477,26 @@ class Loader2DataSet(DataSet):
                 for name, pipe_col in iteritems(columns)}
 
         index = MultiIndex.from_product([self.dates[2:], self.assets])
+
+        def expected_for_col(col):
+            val = vals[col]
+            offset = columns[col].window_length - min_window
+            return concatenate(
+                [
+                    full(offset * index.levshape[1], nan),
+                    full(
+                        (index.levshape[0] - offset) * index.levshape[1],
+                        val,
+                        float,
+                    )
+                ],
+            )
+
         expected = DataFrame(
-            data={col:
-                  concatenate((
-                      full((columns[col].window_length - min_window)
-                           * index.levshape[1],
-                           nan),
-                      full((index.levshape[0]
-                            - (columns[col].window_length - min_window))
-                           * index.levshape[1],
-                           val)))
-                  for col, val in iteritems(vals)},
+            data={col: expected_for_col(col) for col in vals},
             index=index,
-            columns=columns)
+            columns=columns,
+        )
 
         assert_frame_equal(result, expected)
 

tests/pipeline/test_factor.py

@@ -23,7 +23,12 @@
     RSI,
 )
 from zipline.utils.test_utils import check_allclose, check_arrays
-from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype, np_NaT
+from zipline.utils.numpy_utils import (
+    datetime64ns_dtype,
+    float64_dtype,
+    NaTD,
+    NaTns,
+)
 
 from .base import BasePipelineTestCase
 
@@ -309,7 +314,7 @@ def test_masked_rankdata_2d(self,
         mask = eyemask if use_mask else nomask
         if set_missing:
             asfloat[:, 2] = nan
-            asdatetime[:, 2] = np_NaT
+            asdatetime[:, 2] = NaTns
 
         float_result = masked_rankdata_2d(
             data=asfloat,
@@ -321,7 +326,7 @@ def test_masked_rankdata_2d(self,
         datetime_result = masked_rankdata_2d(
             data=asdatetime,
             mask=mask,
-            missing_value=np_NaT,
+            missing_value=NaTns,
             method=method,
             ascending=True,
         )

tests/pipeline/test_numerical_expression.py

@@ -76,9 +76,9 @@ def setUp(self):
         self.h = H()
         self.d = DateFactor()
         self.fake_raw_data = {
-            self.f: full((5, 5), 3),
-            self.g: full((5, 5), 2),
-            self.h: full((5, 5), 1),
+            self.f: full((5, 5), 3, float),
+            self.g: full((5, 5), 2, float),
+            self.h: full((5, 5), 1, float),
             self.d: full((5, 5), 0, dtype='datetime64[ns]'),
         }
         self.mask = DataFrame(True, index=self.dates, columns=self.assets)
@@ -94,7 +94,7 @@ def check_output(self, expr, expected):
 
     def check_constant_output(self, expr, expected):
         self.assertFalse(isnan(expected))
-        return self.check_output(expr, full((5, 5), expected))
+        return self.check_output(expr, full((5, 5), expected, float))
 
     def test_validate_good(self):
         f = self.f
@@ -435,9 +435,9 @@ def test_math_functions(self):
     def test_comparisons(self):
         f, g, h = self.f, self.g, self.h
         self.fake_raw_data = {
-            f: arange(25).reshape(5, 5),
-            g: arange(25).reshape(5, 5) - eye(5),
-            h: full((5, 5), 5),
+            f: arange(25, dtype=float).reshape(5, 5),
+            g: arange(25, dtype=float).reshape(5, 5) - eye(5),
+            h: full((5, 5), 5, dtype=float),
         }
         f_data = self.fake_raw_data[f]
         g_data = self.fake_raw_data[g]
@@ -479,9 +479,9 @@ def test_boolean_binops(self):
         )
 
         self.fake_raw_data = {
-            f: arange(25).reshape(5, 5),
-            g: arange(25).reshape(5, 5) - eye(5),
-            h: full((5, 5), 5),
+            f: arange(25, dtype=float).reshape(5, 5),
+            g: arange(25, dtype=float).reshape(5, 5) - eye(5),
+            h: full((5, 5), 5, dtype=float),
             custom_filter: custom_filter_mask,
         }
 

tests/pipeline/test_term.py

@@ -40,6 +40,7 @@ class SomeFactor(Factor):
     dtype = float64_dtype
     window_length = 5
     inputs = [SomeDataSet.foo, SomeDataSet.bar]
+
 SomeFactorAlias = SomeFactor