Merge pull request numpy#12594 from mattip/pypy3-testing

DEV, BUILD: add pypy3 to azure CI

azure-pipelines.yml

@@ -210,3 +210,16 @@ jobs:
       testResultsFiles: '**/test-*.xml'
       failTaskOnFailedTests: true
       testRunTitle: 'Publish test results for Python $(PYTHON_VERSION) $(BITS)-bit $(TEST_MODE) Windows'
+
+- job: Linux_PyPy3
+  pool:
+    vmIMage: 'ubuntu-16.04'
+  steps:
+  - script: source tools/pypy-test.sh
+    displayName: 'Run PyPy3 Build / Tests'
+    continueOnError: true
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFiles: '**/test-*.xml'
+      testRunTitle: 'Publish test results for PyPy3'
+      failTaskOnFailedTests: true

numpy/core/_internal.py

@@ -8,6 +8,7 @@
 
 import re
 import sys
+import platform
 
 from numpy.compat import unicode
 from .multiarray import dtype, array, ndarray
@@ -16,6 +17,8 @@
 except ImportError:
     ctypes = None
 
+IS_PYPY = platform.python_implementation() == 'PyPy'
+
 if (sys.byteorder == 'little'):
     _nbo = b'<'
 else:
@@ -865,7 +868,12 @@ def npy_ctypes_check(cls):
     try:
         # ctypes class are new-style, so have an __mro__. This probably fails
         # for ctypes classes with multiple inheritance.
-        ctype_base = cls.__mro__[-2]
+        if IS_PYPY:
+            # (..., _ctypes.basics._CData, Bufferable, object)
+            ctype_base = cls.__mro__[-3]
+        else:
+            # # (..., _ctypes._CData, object)
+            ctype_base = cls.__mro__[-2]
         # right now, they're part of the _ctypes module
         return 'ctypes' in ctype_base.__module__
     except Exception:

numpy/core/tests/test_multiarray.py

@@ -36,7 +36,7 @@
     assert_, assert_raises, assert_warns, assert_equal, assert_almost_equal,
     assert_array_equal, assert_raises_regex, assert_array_almost_equal,
     assert_allclose, IS_PYPY, HAS_REFCOUNT, assert_array_less, runstring,
-    temppath, suppress_warnings
+    temppath, suppress_warnings, break_cycles,
     )
 from numpy.core.tests._locales import CommaDecimalPointLocale
 
@@ -131,6 +131,7 @@ def test_writeable_from_buffer(self):
         assert_(vals.flags.writeable)
 
     @pytest.mark.skipif(sys.version_info[0] < 3, reason="Python 2 always copies")
+    @pytest.mark.skipif(IS_PYPY, reason="PyPy always copies")
     def test_writeable_pickle(self):
         import pickle
         # Small arrays will be copied without setting base.
@@ -3783,7 +3784,7 @@ def test_roundtrip(self):
                         a, pickle.loads(pickle.dumps(a, protocol=proto)),
                         err_msg="%r" % a)
             del a, DATA, carray
-            gc.collect()
+            break_cycles()
             # check for reference leaks (gh-12793)
             for ref in refs:
                 assert ref() is None
@@ -7180,7 +7181,7 @@ def test_mem_seteventhook(self):
         # needs to be larger then limit of small memory cacher in ctors.c
         a = np.zeros(1000)
         del a
-        gc.collect()
+        break_cycles()
         _multiarray_tests.test_pydatamem_seteventhook_end()
 
 class TestMapIter(object):
@@ -7752,12 +7753,12 @@ def test_ctypes_data_as_holds_reference(self, arr):
 
         # `ctypes_ptr` should hold onto `arr`
         del arr
-        gc.collect()
+        break_cycles()
         assert_(arr_ref() is not None, "ctypes pointer did not hold onto a reference")
 
         # but when the `ctypes_ptr` object dies, so should `arr`
         del ctypes_ptr
-        gc.collect()
+        break_cycles()
         assert_(arr_ref() is None, "unknowable whether ctypes pointer holds a reference")
 
 
@@ -7939,15 +7940,15 @@ class Dummy(object): pass
         assert_(isinstance(obj_subarray, RaisesInFinalize))
 
         # reference should still be held by obj_arr
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is not None, "object should not already be dead")
 
         del obj_arr
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is not None, "obj_arr should not hold the last reference")
 
         del obj_subarray
-        gc.collect()
+        break_cycles()
         assert_(obj_ref() is None, "no references should remain")
 
 

numpy/core/tests/test_numerictypes.py

@@ -5,7 +5,7 @@
 
 import pytest
 import numpy as np
-from numpy.testing import assert_, assert_equal, assert_raises
+from numpy.testing import assert_, assert_equal, assert_raises, IS_PYPY
 
 # This is the structure of the table used for plain objects:
 #
@@ -491,6 +491,7 @@ def test_issctype(rep, expected):
 
 @pytest.mark.skipif(sys.flags.optimize > 1,
                     reason="no docstrings present to inspect when PYTHONOPTIMIZE/Py_OptimizeFlag > 1")
+@pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
 class TestDocStrings(object):
     def test_platform_dependent_aliases(self):
         if np.int64 is np.int_:

numpy/f2py/tests/test_block_docstring.py

@@ -4,7 +4,7 @@
 import pytest
 from . import util
 
-from numpy.testing import assert_equal
+from numpy.testing import assert_equal, IS_PYPY
 
 class TestBlockDocString(util.F2PyTest):
     code = """
@@ -18,6 +18,7 @@ class TestBlockDocString(util.F2PyTest):
 
     @pytest.mark.skipif(sys.platform=='win32',
                         reason='Fails with MinGW64 Gfortran (Issue #9673)')
+    @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
     def test_block_docstring(self):
         expected = "'i'-array(2,3)\n"
         assert_equal(self.module.block.__doc__, expected)

numpy/lib/tests/test_function_base.py

@@ -11,7 +11,7 @@
 from numpy import ma
 from numpy.testing import (
     assert_, assert_equal, assert_array_equal, assert_almost_equal,
-    assert_array_almost_equal, assert_raises, assert_allclose,
+    assert_array_almost_equal, assert_raises, assert_allclose, IS_PYPY,
     assert_warns, assert_raises_regex, suppress_warnings, HAS_REFCOUNT,
     )
 import numpy.lib.function_base as nfb
@@ -3177,6 +3177,7 @@ def test_string_arg(self):
 class TestAdd_newdoc(object):
 
     @pytest.mark.skipif(sys.flags.optimize == 2, reason="Python running -OO")
+    @pytest.mark.xfail(IS_PYPY, reason="PyPy does not modify tp_doc")
     def test_add_doc(self):
         # test np.add_newdoc
         tgt = "Current flat index into the array."

numpy/testing/_private/utils.py

@@ -6,6 +6,7 @@
 
 import os
 import sys
+import platform
 import re
 import gc
 import operator
@@ -39,6 +40,7 @@
         'SkipTest', 'KnownFailureException', 'temppath', 'tempdir', 'IS_PYPY',
         'HAS_REFCOUNT', 'suppress_warnings', 'assert_array_compare',
         '_assert_valid_refcount', '_gen_alignment_data', 'assert_no_gc_cycles',
+        'break_cycles',
         ]
 
 
@@ -50,7 +52,7 @@ class KnownFailureException(Exception):
 KnownFailureTest = KnownFailureException  # backwards compat
 verbose = 0
 
-IS_PYPY = '__pypy__' in sys.modules
+IS_PYPY = platform.python_implementation() == 'PyPy'
 HAS_REFCOUNT = getattr(sys, 'getrefcount', None) is not None
 
 
@@ -2245,6 +2247,7 @@ def _assert_no_gc_cycles_context(name=None):
 
     # not meaningful to test if there is no refcounting
     if not HAS_REFCOUNT:
+        yield
         return
 
     assert_(gc.isenabled())
@@ -2323,3 +2326,19 @@ def assert_no_gc_cycles(*args, **kwargs):
     args = args[1:]
     with _assert_no_gc_cycles_context(name=func.__name__):
         func(*args, **kwargs)
+
+def break_cycles():
+    """
+    Break reference cycles by calling gc.collect
+    Objects can call other objects' methods (for instance, another object's
+     __del__) inside their own __del__. On PyPy, the interpreter only runs
+    between calls to gc.collect, so multiple calls are needed to completely
+    release all cycles.
+    """
+
+    gc.collect()
+    if IS_PYPY:
+        # interpreter runs now, to call deleted objects' __del__ methods
+        gc.collect()
+        # one more, just to make sure
+        gc.collect()

tools/pypy-test.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Exit if a command fails
+set -e
+set -o pipefail
+# Print expanded commands
+set -x
+
+sudo apt-get -yq update
+sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5
+F77=gfortran-5 F90=gfortran-5 \
+
+# Download the proper OpenBLAS x64 precompiled library
+OPENBLAS=openblas-v0.3.5-manylinux1_x86_64.tar.gz
+echo getting $OPENBLAS
+wget -q https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/$OPENBLAS -O openblas.tar.gz
+mkdir -p openblas
+(cd openblas; tar -xf ../openblas.tar.gz)
+export LD_LIBRARY_PATH=$PWD/openblas/usr/local/lib
+export LIB=$PWD/openblas/usr/local/lib
+export INCLUDE=$PWD/openblas/usr/local/include
+
+# Use a site.cfg to build with local openblas
+cat << EOF > site.cfg
+[openblas]
+libraries = openblas
+library_dirs = $PWD/openblas/usr/local/lib:$LIB
+include_dirs = $PWD/openblas/usr/local/lib:$LIB
+runtime_library_dirs = $PWD/openblas/usr/local/lib
+EOF
+
+echo getting PyPy 3.6 nightly
+wget -q http://buildbot.pypy.org/nightly/py3.6/pypy-c-jit-latest-linux64.tar.bz2 -O pypy.tar.bz2
+mkdir -p pypy3
+(cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2)
+pypy3/bin/pypy3 -mensurepip
+pypy3/bin/pypy3 -m pip install --upgrade pip setuptools
+pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location
+
+echo
+echo pypy3 version 
+pypy3/bin/pypy3 -c "import sys; print(sys.version)"
+echo
+
+pypy3/bin/pypy3 runtests.py --show-build-log -- -rsx \
+      --junitxml=junit/test-results.xml --durations 10
+
+echo Make sure the correct openblas has been linked in
+# rework after merging PR #12790 or alternative
+TEST_GET_CONFIG="import numpy, ctypes, os
+dll = ctypes.CDLL(numpy.core._multiarray_umath.__file__)
+get_config = dll.openblas_get_config
+get_config.restype=ctypes.c_char_p
+res = get_config()
+print('OpenBLAS get_config returned', str(res))
+assert b'OpenBLAS 0.3.5' in res"
+
+pypy3/bin/pip install .
+(cd pypy3; bin/pypy3 -c "$TEST_GET_CONFIG")