merge master into branch

benchmarks/benchmarks/bench_core.py

@@ -162,12 +162,18 @@ def setup(self):
     def time_unpackbits(self):
         np.unpackbits(self.d)
 
+    def time_unpackbits_little(self):
+        np.unpackbits(self.d, bitorder="little")
+
     def time_unpackbits_axis0(self):
         np.unpackbits(self.d2, axis=0)
 
     def time_unpackbits_axis1(self):
         np.unpackbits(self.d2, axis=1)
 
+    def time_unpackbits_axis1_little(self):
+        np.unpackbits(self.d2, bitorder="little", axis=1)
+
 
 class Indices(Benchmark):
     def time_indices(self):

doc/neps/nep-0019-rng-policy.rst

@@ -6,6 +6,7 @@ NEP 19 — Random Number Generator Policy
 :Status: Accepted
 :Type: Standards Track
 :Created: 2018-05-24
+:Updated: 2019-05-21
 :Resolution: https://mail.python.org/pipermail/numpy-discussion/2018-June/078126.html
 
 Abstract
@@ -91,12 +92,13 @@ those contributors simply walked away.
 Implementation
 --------------
 
-Work on a proposed new pseudorandom number generator (PRNG) subsystem is
-already underway in the randomgen_ project.  The specifics of the new design
-are out of scope for this NEP and up for much discussion, but we will discuss
-general policies that will guide the evolution of whatever code is adopted.  We
-will also outline just a few of the requirements that such a new system must
-have to support the policy proposed in this NEP.
+Work on a proposed new Pseudo Random Number Generator (PRNG) subsystem is
+already underway in the randomgen_
+project.  The specifics of the new design are out of scope for this NEP and up
+for much discussion, but we will discuss general policies that will guide the
+evolution of whatever code is adopted.  We will also outline just a few of the
+requirements that such a new system must have to support the policy proposed in
+this NEP.
 
 First, we will maintain API source compatibility just as we do with the rest of
 ``numpy``.  If we *must* make a breaking change, we will only do so with an
@@ -119,21 +121,22 @@ Gaussian variate generation to the faster `Ziggurat algorithm
 discouraged improvement would be tweaking the Ziggurat tables just a little bit
 for a small performance improvement.
 
-Any new design for the RNG subsystem will provide a choice of different core
+Any new design for the random subsystem will provide a choice of different core
 uniform PRNG algorithms.  A promising design choice is to make these core
 uniform PRNGs their own lightweight objects with a minimal set of methods
 (randomgen_ calls them “BitGenerators”).  The broader set of non-uniform
-distributions will be its own class "Generator" that holds a reference to one
-of these BitGenerator objects and simply delegates to the BitGenerator object
-when it needs uniform random numbers.  To borrow an example from randomgen_, the
+distributions will be its own class that holds a reference to one of these core
+uniform PRNG objects and simply delegates to the core uniform PRNG object when
+it needs uniform random numbers (randomgen_ calls this the Generator).  To
+borrow an example from randomgen_, the
 class ``MT19937`` is a BitGenerator that implements the classic Mersenne Twister
 algorithm.  The class ``Generator`` wraps around the BitGenerator to provide
 all of the non-uniform distribution methods::
 
     # This is not the only way to instantiate this object.
     # This is just handy for demonstrating the delegation.
-    >>> brng = MT19937(seed)
-    >>> rg = Generator(brng)
+    >>> bg = MT19937(seed)
+    >>> rg = Generator(bg)
     >>> x = rg.standard_normal(10)
 
 We will be more strict about a select subset of methods on these BitGenerator
@@ -143,14 +146,14 @@ distributions and which are needed to abstract over the implementation details
 of the variety of BitGenerator algorithms.  Namely,
 
     * ``.bytes()``
-    * ``.integers()`` (which replaces ``randint`` and ``random_integers``
-    * ``.random()`` (which replaces ``.random_sample()``
+    * ``integers()`` (formerly ``.random_integers()``)
+    * ``random()`` (formerly ``.random_sample()``)
 
 The distributions class (``Generator``) SHOULD have all of the same
 distribution methods as ``RandomState`` with close-enough function signatures
 such that almost all code that currently works with ``RandomState`` instances
-will work with ``Generator`` instances (ignoring the precise stream values).
-Some variance will be allowed for integer distributions: in order to
+will work with ``Generator`` instances (ignoring the precise stream
+values).  Some variance will be allowed for integer distributions: in order to
 avoid some of the cross-platform problems described above, these SHOULD be
 rewritten to work with ``uint64`` numbers on all platforms.
 
@@ -185,12 +188,13 @@ This legacy distributions class MUST be accessible under the name
 instantiating ``numpy.random.RandomState`` with a given state should
 instantiate the Mersenne Twister BitGenerator with the same state.  The legacy
 distributions class MUST be capable of accepting other BitGenerators.  The
-purpose here is to ensure that one can write a program with a consistent
-BitGenerator state with a mixture of libraries that may or may not have
-upgraded from ``RandomState``.  Instances of the legacy distributions class
-MUST respond ``True`` to ``isinstance(rg, numpy.random.RandomState)`` because
-there is current utility code that relies on that check.  Similarly, old
-pickles of ``numpy.random.RandomState`` instances MUST unpickle correctly.
+purpose
+here is to ensure that one can write a program with a consistent BitGenerator
+state with a mixture of libraries that may or may not have upgraded from
+``RandomState``.  Instances of the legacy distributions class MUST respond
+``True`` to ``isinstance(rg, numpy.random.RandomState)`` because there is
+current utility code that relies on that check.  Similarly, old pickles of
+``numpy.random.RandomState`` instances MUST unpickle correctly.
 
 
 ``numpy.random.*``
@@ -209,7 +213,8 @@ consistently and usefully, but a very common usage is in unit tests where many
 of the problems of global state are less likely.
 
 This NEP does not propose removing these functions or changing them to use the
-less-stable ``Generator`` distribution implementations.  Future NEPs might.
+less-stable ``Generator`` distribution implementations.  Future NEPs
+might.
 
 Specifically, the initial release of the new PRNG subsystem SHALL leave these
 convenience functions as aliases to the methods on a global ``RandomState``
@@ -224,11 +229,11 @@ BitGenerator underneath the global ``RandomState`` with any other BitGenerator
 object (we leave the precise API details up to the new subsystem).  Calling
 ``numpy.random.seed()`` thereafter SHOULD just pass the given seed to the
 current BitGenerator object and not attempt to reset the BitGenerator to the
-Mersenne Twister.  The set of ``numpy.random.*`` convenience functions SHALL remain the
-same as they currently are.  They SHALL be aliases to the ``RandomState``
-methods and not the new less-stable distributions class (``Generator``, in the
-examples above). Users who want to get the fastest, best distributions can
-follow best practices and instantiate Generator objects explicitly.
+Mersenne Twister.  The set of ``numpy.random.*`` convenience functions SHALL
+remain the same as they currently are.  They SHALL be aliases to the
+``RandomState`` methods and not the new less-stable distributions class
+(``Generator``, in the examples above). Users who want to get the fastest, best
+distributions can follow best practices and instantiate generator objects explicitly.
 
 This NEP does not propose that these requirements remain in perpetuity.  After
 we have experience with the new PRNG subsystem, we can and should revisit these
@@ -309,7 +314,7 @@ The current proposal solves all of these problems.  All current usages of
 discouraged through documentation.  Unit tests can continue to use the full
 complement of ``RandomState`` methods.  Mixed ``RandomState/Generator``
 code can safely share the common BitGenerator state.  Unmodified ``RandomState``
-code can make use of the new features of alternative BitGenerators like settable
+code can make use of the new features of alternative BitGenerator-like settable
 streams.
 
 

doc/release/1.17.0-notes.rst

@@ -33,6 +33,16 @@ The internal use of these functions has been refactored and there are better
 alternatives. Relace ``exec_command`` with `subprocess.Popen` and
 ``temp_file_name`` with `tempfile.mkstemp`.
 
+Writeable flag of C-API wrapped arrays
+--------------------------------------
+When an array is created from the C-API to wrap a pointer to data, the only
+indication we have of the read-write nature of the data is the ``writeable``
+flag set during creation. It is dangerous to force the flag to writeable.
+In the future it will not be possible to switch the writeable flag to ``True``
+from python.
+This deprecation should not affect many users since arrays created in such
+a manner are very rare in practice and only available through the NumPy C-API.
+
 
 Future Changes
 ==============

numpy/core/_add_newdocs.py

@@ -4618,10 +4618,12 @@
         number of outputs; use `None` for uninitialized outputs to be
         allocated by the ufunc.
     where : array_like, optional
-        Values of True indicate to calculate the ufunc at that position, values
-        of False indicate to leave the value in the output alone.  Note that if
-        an uninitialized return array is created via the default ``out=None``,
-        then the elements where the values are False will remain uninitialized.
+        This condition is broadcast over the input. At locations where the
+        condition is True, the `out` array will be set to the ufunc result.
+        Elsewhere, the `out` array will retain its original value.
+        Note that if an uninitialized `out` array is created via the default
+        ``out=None``, locations within it where the condition is False will
+        remain uninitialized.
     **kwargs
         For other keyword-only arguments, see the :ref:`ufunc docs <ufuncs.kwargs>`.
 

numpy/core/code_generators/ufunc_docstrings.py

@@ -26,8 +26,12 @@ def get(name):
             a freshly-allocated array is returned. A tuple (possible only as a
             keyword argument) must have length equal to the number of outputs.
         where : array_like, optional
-            Values of True indicate to calculate the ufunc at that position, values
-            of False indicate to leave the value in the output alone.
+            This condition is broadcast over the input. At locations where the
+            condition is True, the `out` array will be set to the ufunc result.
+            Elsewhere, the `out` array will retain its original value.
+            Note that if an uninitialized `out` array is created via the default
+            ``out=None``, locations within it where the condition is False will
+            remain uninitialized.
         **kwargs
             For other keyword-only arguments, see the
             :ref:`ufunc docs <ufuncs.kwargs>`.

numpy/core/fromnumeric.py

@@ -2145,7 +2145,7 @@ def sum(a, axis=None, dtype=None, out=None, keepdims=np._NoValue,
         warnings.warn(
             "Calling np.sum(generator) is deprecated, and in the future will give a different result. "
             "Use np.sum(np.fromiter(generator)) or the python sum builtin instead.",
-            DeprecationWarning, stacklevel=2)
+            DeprecationWarning, stacklevel=3)
 
         res = _sum_(a)
         if out is not None:
@@ -3569,5 +3569,5 @@ def rank(a):
     warnings.warn(
         "`rank` is deprecated; use the `ndim` attribute or function instead. "
         "To find the rank of a matrix see `numpy.linalg.matrix_rank`.",
-        VisibleDeprecationWarning, stacklevel=2)
+        VisibleDeprecationWarning, stacklevel=3)
     return ndim(a)

numpy/core/multiarray.py

@@ -1116,7 +1116,7 @@ def putmask(a, mask, values):
 @array_function_from_c_func_and_dispatcher(_multiarray_umath.packbits)
 def packbits(a, axis=None, bitorder='big'):
     """
-    packbits(a, axis=None)
+    packbits(a, axis=None, bitorder='big')
 
     Packs the elements of a binary-valued array into bits in a uint8 array.
 
@@ -1174,7 +1174,7 @@ def packbits(a, axis=None, bitorder='big'):
 @array_function_from_c_func_and_dispatcher(_multiarray_umath.unpackbits)
 def unpackbits(a, axis=None, count=None, bitorder='big'):
     """
-    unpackbits(a, axis=None, count=None)
+    unpackbits(a, axis=None, count=None, bitorder='big')
 
     Unpacks elements of a uint8 array into a binary-valued output array.
 

numpy/core/numeric.py

@@ -1139,7 +1139,7 @@ def roll(a, shift, axis=None):
     array([8, 9, 0, 1, 2, 3, 4, 5, 6, 7])
     >>> np.roll(x, -2)
     array([2, 3, 4, 5, 6, 7, 8, 9, 0, 1])
-    
+
     >>> x2 = np.reshape(x, (2,5))
     >>> x2
     array([[0, 1, 2, 3, 4],
@@ -1606,11 +1606,11 @@ def cross(a, b, axisa=-1, axisb=-1, axisc=-1, axis=None):
 
 
 @set_module('numpy')
-def indices(dimensions, dtype=int):
+def indices(dimensions, dtype=int, sparse=False):
     """
     Return an array representing the indices of a grid.
 
-    Compute an array where the subarrays contain index values 0,1,...
+    Compute an array where the subarrays contain index values 0, 1, ...
     varying only along the corresponding axis.
 
     Parameters
@@ -1619,28 +1619,38 @@ def indices(dimensions, dtype=int):
         The shape of the grid.
     dtype : dtype, optional
         Data type of the result.
+    sparse : boolean, optional
+        Return a sparse representation of the grid instead of a dense
+        representation. Default is False.
+
+        .. versionadded:: 1.17
 
     Returns
     -------
-    grid : ndarray
-        The array of grid indices,
-        ``grid.shape = (len(dimensions),) + tuple(dimensions)``.
+    grid : one ndarray or tuple of ndarrays
+        If sparse is False:
+            Returns one array of grid indices,
+            ``grid.shape = (len(dimensions),) + tuple(dimensions)``.
+        If sparse is True:
+            Returns a tuple of arrays, with
+            ``grid[i].shape = (1, ..., 1, dimensions[i], 1, ..., 1)`` with
+            dimensions[i] in the ith place
 
     See Also
     --------
-    mgrid, meshgrid
+    mgrid, ogrid, meshgrid
 
     Notes
     -----
-    The output shape is obtained by prepending the number of dimensions
-    in front of the tuple of dimensions, i.e. if `dimensions` is a tuple
-    ``(r0, ..., rN-1)`` of length ``N``, the output shape is
-    ``(N,r0,...,rN-1)``.
+    The output shape in the dense case is obtained by prepending the number
+    of dimensions in front of the tuple of dimensions, i.e. if `dimensions`
+    is a tuple ``(r0, ..., rN-1)`` of length ``N``, the output shape is
+    ``(N, r0, ..., rN-1)``.
 
     The subarrays ``grid[k]`` contains the N-D array of indices along the
     ``k-th`` axis. Explicitly::
 
-        grid[k,i0,i1,...,iN-1] = ik
+        grid[k, i0, i1, ..., iN-1] = ik
 
     Examples
     --------
@@ -1665,15 +1675,36 @@ def indices(dimensions, dtype=int):
     Note that it would be more straightforward in the above example to
     extract the required elements directly with ``x[:2, :3]``.
 
+    If sparse is set to true, the grid will be returned in a sparse
+    representation.
+
+    >>> i, j = np.indices((2, 3), sparse=True)
+    >>> i.shape
+    (2, 1)
+    >>> j.shape
+    (1, 3)
+    >>> i        # row indices
+    array([[0],
+           [1]])
+    >>> j        # column indices
+    array([[0, 1, 2]])
+
     """
     dimensions = tuple(dimensions)
     N = len(dimensions)
     shape = (1,)*N
-    res = empty((N,)+dimensions, dtype=dtype)
+    if sparse:
+        res = tuple()
+    else:
+        res = empty((N,)+dimensions, dtype=dtype)
     for i, dim in enumerate(dimensions):
-        res[i] = arange(dim, dtype=dtype).reshape(
+        idx = arange(dim, dtype=dtype).reshape(
             shape[:i] + (dim,) + shape[i+1:]
         )
+        if sparse:
+            res = res + (idx,)
+        else:
+            res[i] = idx
     return res
 
 

numpy/core/src/multiarray/_multiarray_tests.c.src

@@ -847,6 +847,29 @@ get_buffer_info(PyObject *NPY_UNUSED(self), PyObject *args)
 
 #undef GET_PYBUF_FLAG
 
+/*
+ * Return a new array object wrapping existing C-allocated (dummy) data.
+ * Such an array does not own its data (must not free it), but because it
+ * wraps C data, it also has no base object. Used to test arr.flags.writeable
+ * setting behaviour.
+ */
+static PyObject*
+get_c_wrapping_array(PyObject* NPY_UNUSED(self), PyObject* arg)
+{
+    int writeable, flags;
+    npy_intp zero = 0;
+
+    writeable = PyObject_IsTrue(arg);
+    if (error_converting(writeable)) {
+        return NULL;
+    }
+
+    flags = writeable ? NPY_ARRAY_WRITEABLE : 0;
+    /* Create an empty array (which points to a random place) */
+    return PyArray_NewFromDescr(&PyArray_Type, PyArray_DescrFromType(NPY_INTP),
+                                1, &zero, NULL, &zero, flags, NULL);
+}
+
 
 /*
  * Test C-api level item getting.
@@ -1932,6 +1955,9 @@ static PyMethodDef Multiarray_TestsMethods[] = {
     {"get_buffer_info",
         get_buffer_info,
         METH_VARARGS, NULL},
+    {"get_c_wrapping_array",
+        get_c_wrapping_array,
+        METH_O, NULL},
     {"array_indexing",
         array_indexing,
         METH_VARARGS, NULL},