From fc1bb6b5b751dc20d7a4fe85ec4fd4287968a0ab Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Thu, 2 Nov 2023 14:08:05 +0100 Subject: [PATCH 01/10] API: NumPy NPY_MAXDIMS and NPY_MAXARGS to 64 introduce NPY_AXIS_RAVEL Bumping these two has three main caveats: 1. We cannot break ABI for the iterator macros, which means the iterators now need to use 32/their own macro. 2. We used axis=MAXDIMS to mean axis=None, introduce NPY_AXIS_RAVEL to replace this, it will be run-time outside NumPy. 3. The old style iterators cannot deal with high dimensions, meaning that some functions just won't work (because they use them). --- doc/source/reference/c-api/array.rst | 5 +- .../reference/c-api/types-and-structures.rst | 39 ++++++++------- numpy/__init__.cython-30.pxd | 5 -- numpy/__init__.pxd | 5 -- numpy/_core/include/numpy/ndarrayobject.h | 1 - numpy/_core/include/numpy/ndarraytypes.h | 49 +++++++++++-------- numpy/_core/include/numpy/npy_2_compat.h | 10 ++++ .../src/multiarray/_multiarray_tests.c.src | 14 ++++-- numpy/_core/src/multiarray/calculation.c | 2 +- numpy/_core/src/multiarray/compiled_base.c | 4 +- numpy/_core/src/multiarray/conversion_utils.c | 13 +---- numpy/_core/src/multiarray/ctors.c | 4 +- numpy/_core/src/multiarray/iterators.c | 8 +++ numpy/_core/src/multiarray/methods.c | 14 +++--- numpy/_core/src/multiarray/multiarraymodule.c | 2 +- numpy/_core/src/multiarray/refcount.c | 14 ++++++ numpy/_core/src/multiarray/sequence.c | 2 +- numpy/_core/tests/test_conversion_utils.py | 4 +- 18 files changed, 112 insertions(+), 83 deletions(-) diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index d1a794c13879..18be77512b65 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3411,7 +3411,10 @@ Other constants .. c:macro:: NPY_MAXDIMS - The maximum number of dimensions allowed in arrays. + .. note:: + NumPy used to have a maximum number of dimensions set to 32 and now 64. + This was/is mostly useful for stack allocations. We now ask you to + define (and test) such a limit in your own project if needed. .. c:macro:: NPY_MAXARGS diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index 3d5f9fe0b61c..b6b00bc9f19c 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -119,8 +119,11 @@ PyArray_Type and PyArrayObject array. When nd is 0, the array is sometimes called a rank-0 array. Such arrays have undefined dimensions and strides and cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in - ``ndarraytypes.h`` points to this data member. :c:data:`NPY_MAXDIMS` - is the largest number of dimensions for any array. + ``ndarraytypes.h`` points to this data member. + Although most operations may be limited in dimensionality, we do not + advertise a maximum dimension. Anyone explicitly relying on one + must check for it. Before NumPy 2.0, NumPy used 32 dimensions at most + after it, the limit is currently 64. .. c:member:: npy_intp *dimensions @@ -986,11 +989,11 @@ PyArrayIter_Type and PyArrayIterObject int nd_m1; npy_intp index; npy_intp size; - npy_intp coordinates[NPY_MAXDIMS]; - npy_intp dims_m1[NPY_MAXDIMS]; - npy_intp strides[NPY_MAXDIMS]; - npy_intp backstrides[NPY_MAXDIMS]; - npy_intp factors[NPY_MAXDIMS]; + npy_intp coordinates[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp dims_m1[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp strides[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp backstrides[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp factors[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayObject *ao; char *dataptr; npy_bool contiguous; @@ -1086,8 +1089,8 @@ PyArrayMultiIter_Type and PyArrayMultiIterObject npy_intp size; npy_intp index; int nd; - npy_intp dimensions[NPY_MAXDIMS]; - PyArrayIterObject *iters[NPY_MAXDIMS]; + npy_intp dimensions[NPY_MAXDIMS_LEGACY_ITERS]; + PyArrayIterObject *iters[NPY_MAXDIMS_LEGACY_ITERS]; } PyArrayMultiIterObject; .. c:macro: PyObject_HEAD @@ -1141,20 +1144,20 @@ PyArrayNeighborhoodIter_Type and PyArrayNeighborhoodIterObject PyObject_HEAD int nd_m1; npy_intp index, size; - npy_intp coordinates[NPY_MAXDIMS] - npy_intp dims_m1[NPY_MAXDIMS]; - npy_intp strides[NPY_MAXDIMS]; - npy_intp backstrides[NPY_MAXDIMS]; - npy_intp factors[NPY_MAXDIMS]; + npy_intp coordinates[NPY_MAXDIMS_LEGACY_ITERS] + npy_intp dims_m1[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp strides[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp backstrides[NPY_MAXDIMS_LEGACY_ITERS]; + npy_intp factors[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayObject *ao; char *dataptr; npy_bool contiguous; - npy_intp bounds[NPY_MAXDIMS][2]; - npy_intp limits[NPY_MAXDIMS][2]; - npy_intp limits_sizes[NPY_MAXDIMS]; + npy_intp bounds[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits_sizes[NPY_MAXDIMS_LEGACY_ITERS]; npy_iter_get_dataptr_t translate; npy_intp nd; - npy_intp dimensions[NPY_MAXDIMS]; + npy_intp dimensions[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayIterObject* _internal_iter; char* constant; int mode; diff --git a/numpy/__init__.cython-30.pxd b/numpy/__init__.cython-30.pxd index 71767ae4586e..6b3d57a31f01 100644 --- a/numpy/__init__.cython-30.pxd +++ b/numpy/__init__.cython-30.pxd @@ -186,11 +186,6 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_UPDATE_ALL - cdef enum: - NPY_MAXDIMS - - npy_intp NPY_MAX_ELSIZE - ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) ctypedef struct PyArray_ArrayDescr: diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index 153a5b8c6885..17a29f3979aa 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -181,11 +181,6 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_UPDATE_ALL - cdef enum: - NPY_MAXDIMS - - npy_intp NPY_MAX_ELSIZE - ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) ctypedef struct PyArray_ArrayDescr: diff --git a/numpy/_core/include/numpy/ndarrayobject.h b/numpy/_core/include/numpy/ndarrayobject.h index 39b56d9051b4..ad513a464d74 100644 --- a/numpy/_core/include/numpy/ndarrayobject.h +++ b/numpy/_core/include/numpy/ndarrayobject.h @@ -103,7 +103,6 @@ extern "C" { #define PyArray_FILLWBYTE(obj, val) memset(PyArray_DATA(obj), val, \ PyArray_NBYTES(obj)) -#define NPY_MAX_ELSIZE (2 * NPY_SIZEOF_LONGDOUBLE) #define PyArray_ContiguousFromAny(op, type, min_depth, max_depth) \ PyArray_FromAny(op, PyArray_DescrFromType(type), min_depth, \ diff --git a/numpy/_core/include/numpy/ndarraytypes.h b/numpy/_core/include/numpy/ndarraytypes.h index 3ff8c239bb94..e420441571d2 100644 --- a/numpy/_core/include/numpy/ndarraytypes.h +++ b/numpy/_core/include/numpy/ndarraytypes.h @@ -35,10 +35,17 @@ * The array creation itself could have arbitrary dimensions but all * the places where static allocation is used would need to be changed * to dynamic (including inside of several structures) + * + * As of NumPy 2.0, we strongly discourage the downstream use of NPY_MAXDIMS, + * but since auditing everything seems a big ask, define it as 64. + * A future version could: + * - Increase or remove the limit and require recompilation (like 2.0 did) + * - Deprecate or remove the macro but keep the limit (at basically any time) */ - -#define NPY_MAXDIMS 32 -#define NPY_MAXARGS 32 +#define NPY_MAXDIMS 64 +/* We cannot change this as it would break ABI: */ +#define NPY_MAXDIMS_LEGACY_ITERS 32 +#define NPY_MAXARGS 64 /* Used for Converter Functions "O&" code in ParseTuple */ #define NPY_FAIL 0 @@ -1095,18 +1102,18 @@ struct PyArrayIterObject_tag { PyObject_HEAD int nd_m1; /* number of dimensions - 1 */ npy_intp index, size; - npy_intp coordinates[NPY_MAXDIMS];/* N-dimensional loop */ - npy_intp dims_m1[NPY_MAXDIMS]; /* ao->dimensions - 1 */ - npy_intp strides[NPY_MAXDIMS]; /* ao->strides or fake */ - npy_intp backstrides[NPY_MAXDIMS];/* how far to jump back */ - npy_intp factors[NPY_MAXDIMS]; /* shape factors */ + npy_intp coordinates[NPY_MAXDIMS_LEGACY_ITERS];/* N-dimensional loop */ + npy_intp dims_m1[NPY_MAXDIMS_LEGACY_ITERS]; /* ao->dimensions - 1 */ + npy_intp strides[NPY_MAXDIMS_LEGACY_ITERS]; /* ao->strides or fake */ + npy_intp backstrides[NPY_MAXDIMS_LEGACY_ITERS];/* how far to jump back */ + npy_intp factors[NPY_MAXDIMS_LEGACY_ITERS]; /* shape factors */ PyArrayObject *ao; char *dataptr; /* pointer to current item*/ npy_bool contiguous; - npy_intp bounds[NPY_MAXDIMS][2]; - npy_intp limits[NPY_MAXDIMS][2]; - npy_intp limits_sizes[NPY_MAXDIMS]; + npy_intp bounds[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits_sizes[NPY_MAXDIMS_LEGACY_ITERS]; npy_iter_get_dataptr_t translate; } ; @@ -1230,7 +1237,7 @@ typedef struct { npy_intp size; /* broadcasted size */ npy_intp index; /* current index */ int nd; /* number of dims */ - npy_intp dimensions[NPY_MAXDIMS]; /* dimensions */ + npy_intp dimensions[NPY_MAXDIMS_LEGACY_ITERS]; /* dimensions */ PyArrayIterObject *iters[NPY_MAXARGS]; /* iterators */ } PyArrayMultiIterObject; @@ -1335,18 +1342,18 @@ typedef struct { */ int nd_m1; /* number of dimensions - 1 */ npy_intp index, size; - npy_intp coordinates[NPY_MAXDIMS];/* N-dimensional loop */ - npy_intp dims_m1[NPY_MAXDIMS]; /* ao->dimensions - 1 */ - npy_intp strides[NPY_MAXDIMS]; /* ao->strides or fake */ - npy_intp backstrides[NPY_MAXDIMS];/* how far to jump back */ - npy_intp factors[NPY_MAXDIMS]; /* shape factors */ + npy_intp coordinates[NPY_MAXDIMS_LEGACY_ITERS];/* N-dimensional loop */ + npy_intp dims_m1[NPY_MAXDIMS_LEGACY_ITERS]; /* ao->dimensions - 1 */ + npy_intp strides[NPY_MAXDIMS_LEGACY_ITERS]; /* ao->strides or fake */ + npy_intp backstrides[NPY_MAXDIMS_LEGACY_ITERS];/* how far to jump back */ + npy_intp factors[NPY_MAXDIMS_LEGACY_ITERS]; /* shape factors */ PyArrayObject *ao; char *dataptr; /* pointer to current item*/ npy_bool contiguous; - npy_intp bounds[NPY_MAXDIMS][2]; - npy_intp limits[NPY_MAXDIMS][2]; - npy_intp limits_sizes[NPY_MAXDIMS]; + npy_intp bounds[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits[NPY_MAXDIMS_LEGACY_ITERS][2]; + npy_intp limits_sizes[NPY_MAXDIMS_LEGACY_ITERS]; npy_iter_get_dataptr_t translate; /* @@ -1355,7 +1362,7 @@ typedef struct { npy_intp nd; /* Dimensions is the dimension of the array */ - npy_intp dimensions[NPY_MAXDIMS]; + npy_intp dimensions[NPY_MAXDIMS_LEGACY_ITERS]; /* * Neighborhood points coordinates are computed relatively to the diff --git a/numpy/_core/include/numpy/npy_2_compat.h b/numpy/_core/include/numpy/npy_2_compat.h index 9dbc16e9d4bb..fcda1c69c674 100644 --- a/numpy/_core/include/numpy/npy_2_compat.h +++ b/numpy/_core/include/numpy/npy_2_compat.h @@ -52,4 +52,14 @@ #endif +#if NPY_FEATURE_VERSION >= NPY_2_0_API_VERSION + #define NPY_RAVEL_AXIS NPY_MIN_INT +#elif NPY_ABI_VERSION < 0x02000000 + #define NPY_RAVEL_AXIS 32 +#else + #define NPY_RAVEL_AXIS \ + (PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION ? -1 : 32) +#endif + + #endif /* NUMPY_CORE_INCLUDE_NUMPY_NPY_2_COMPAT_H_ */ diff --git a/numpy/_core/src/multiarray/_multiarray_tests.c.src b/numpy/_core/src/multiarray/_multiarray_tests.c.src index c5597b6da5a7..bf7381f56023 100644 --- a/numpy/_core/src/multiarray/_multiarray_tests.c.src +++ b/numpy/_core/src/multiarray/_multiarray_tests.c.src @@ -84,7 +84,7 @@ static int copy_@name@(PyArrayIterObject *itx, PyArrayNeighborhoodIterObject *ni { npy_intp i, j; @type@ *ptr; - npy_intp odims[NPY_MAXDIMS]; + npy_intp odims[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayObject *aout; /* @@ -125,7 +125,7 @@ static int copy_object(PyArrayIterObject *itx, PyArrayNeighborhoodIterObject *ni PyObject **out) { npy_intp i, j; - npy_intp odims[NPY_MAXDIMS]; + npy_intp odims[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayObject *aout; PyArray_CopySwapFunc *copyswap = PyArray_DESCR(itx->ao)->f->copyswap; npy_int itemsize = PyArray_ITEMSIZE(itx->ao); @@ -166,7 +166,7 @@ test_neighborhood_iterator(PyObject* NPY_UNUSED(self), PyObject* args) PyArrayIterObject *itx; int i, typenum, mode, st; Py_ssize_t idxstart = 0; - npy_intp bounds[NPY_MAXDIMS*2]; + npy_intp bounds[NPY_MAXDIMS_LEGACY_ITERS*2]; PyArrayNeighborhoodIterObject *niterx; if (!PyArg_ParseTuple(args, "OOOi|n", &x, &b, &fill, &mode, &idxstart)) { @@ -298,7 +298,7 @@ copy_double_double(PyArrayNeighborhoodIterObject *itx, { npy_intp i, j; double *ptr; - npy_intp odims[NPY_MAXDIMS]; + npy_intp odims[NPY_MAXDIMS_LEGACY_ITERS]; PyArrayObject *aout; /* @@ -338,7 +338,7 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) PyArrayObject *ax; PyArrayIterObject *itx; int i, typenum, mode1, mode2, st; - npy_intp bounds[NPY_MAXDIMS*2]; + npy_intp bounds[NPY_MAXDIMS_LEGACY_ITERS*2]; PyArrayNeighborhoodIterObject *niterx1, *niterx2; if (!PyArg_ParseTuple(args, "OOiOi", &x, &b1, &mode1, &b2, &mode2)) { @@ -358,6 +358,10 @@ test_neighborhood_iterator_oob(PyObject* NPY_UNUSED(self), PyObject* args) if (ax == NULL) { return NULL; } + if (PyArray_NDIM(ax) > NPY_MAXDIMS_LEGACY_ITERS) { + PyErr_SetString(PyExc_TypeError, "too many dimensions."); + goto clean_ax; + } if (PySequence_Size(b1) != 2 * PyArray_NDIM(ax)) { PyErr_SetString(PyExc_ValueError, "bounds sequence 1 size not compatible with x input"); diff --git a/numpy/_core/src/multiarray/calculation.c b/numpy/_core/src/multiarray/calculation.c index 145bfa012e38..6c85837cb0bc 100644 --- a/numpy/_core/src/multiarray/calculation.c +++ b/numpy/_core/src/multiarray/calculation.c @@ -107,7 +107,7 @@ _PyArray_ArgMinMaxCommon(PyArrayObject *op, } else { out_shape = _shape_buf; - if (axis_copy == NPY_MAXDIMS) { + if (axis_copy == NPY_RAVEL_AXIS) { for (int i = 0; i < out_ndim; i++) { out_shape[i] = 1; } diff --git a/numpy/_core/src/multiarray/compiled_base.c b/numpy/_core/src/multiarray/compiled_base.c index 87f0ba549b1b..2c4e6c7e0fdf 100644 --- a/numpy/_core/src/multiarray/compiled_base.c +++ b/numpy/_core/src/multiarray/compiled_base.c @@ -1977,7 +1977,7 @@ NPY_NO_EXPORT PyObject * io_pack(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { PyObject *obj; - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; static char *kwlist[] = {"in", "axis", "bitorder", NULL}; char c = 'b'; const char * order_str = NULL; @@ -2005,7 +2005,7 @@ NPY_NO_EXPORT PyObject * io_unpack(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwds) { PyObject *obj; - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyObject *count = Py_None; static char *kwlist[] = {"in", "axis", "count", "bitorder", NULL}; const char * c = NULL; diff --git a/numpy/_core/src/multiarray/conversion_utils.c b/numpy/_core/src/multiarray/conversion_utils.c index 1a6fd118a29b..52ad44c20621 100644 --- a/numpy/_core/src/multiarray/conversion_utils.c +++ b/numpy/_core/src/multiarray/conversion_utils.c @@ -183,7 +183,7 @@ PyArray_IntpConverter(PyObject *obj, PyArray_Dims *seq) if (len > NPY_MAXDIMS) { PyErr_Format(PyExc_ValueError, "maximum supported dimension for an ndarray " - "is %d, found %d", NPY_MAXDIMS, len); + "is currently %d, found %d", NPY_MAXDIMS, len); Py_DECREF(seq_obj); return NPY_FAIL; } @@ -325,7 +325,7 @@ NPY_NO_EXPORT int PyArray_AxisConverter(PyObject *obj, int *axis) { if (obj == Py_None) { - *axis = NPY_MAXDIMS; + *axis = NPY_RAVEL_AXIS; } else { *axis = PyArray_PyIntAsInt_ErrMsg(obj, @@ -333,15 +333,6 @@ PyArray_AxisConverter(PyObject *obj, int *axis) if (error_converting(*axis)) { return NPY_FAIL; } - if (*axis == NPY_MAXDIMS){ - /* NumPy 1.23, 2022-05-19 */ - if (DEPRECATE("Using `axis=32` (MAXDIMS) is deprecated. " - "32/MAXDIMS had the same meaning as `axis=None` which " - "should be used instead. " - "(Deprecated NumPy 1.23)") < 0) { - return NPY_FAIL; - } - } } return NPY_SUCCEED; } diff --git a/numpy/_core/src/multiarray/ctors.c b/numpy/_core/src/multiarray/ctors.c index 8aaeb5b39000..ac97fdd69b3b 100644 --- a/numpy/_core/src/multiarray/ctors.c +++ b/numpy/_core/src/multiarray/ctors.c @@ -2785,14 +2785,14 @@ PyArray_CheckAxis(PyArrayObject *arr, int *axis, int flags) PyObject *temp1, *temp2; int n = PyArray_NDIM(arr); - if (*axis == NPY_MAXDIMS || n == 0) { + if (*axis == NPY_RAVEL_AXIS || n == 0) { if (n != 1) { temp1 = PyArray_Ravel(arr,0); if (temp1 == NULL) { *axis = 0; return NULL; } - if (*axis == NPY_MAXDIMS) { + if (*axis == NPY_RAVEL_AXIS) { *axis = PyArray_NDIM((PyArrayObject *)temp1)-1; } } diff --git a/numpy/_core/src/multiarray/iterators.c b/numpy/_core/src/multiarray/iterators.c index 81f8a52d743f..0342bc7aa03c 100644 --- a/numpy/_core/src/multiarray/iterators.c +++ b/numpy/_core/src/multiarray/iterators.c @@ -133,6 +133,8 @@ PyArray_RawIterBaseInit(PyArrayIterObject *it, PyArrayObject *ao) int nd, i; nd = PyArray_NDIM(ao); + /* The legacy iterator only supports 32 dimensions */ + assert(nd <= NPY_MAXDIMS_LEGACY_ITERS); PyArray_UpdateFlags(ao, NPY_ARRAY_C_CONTIGUOUS); if (PyArray_ISCONTIGUOUS(ao)) { it->contiguous = 1; @@ -190,6 +192,12 @@ PyArray_IterNew(PyObject *obj) return NULL; } ao = (PyArrayObject *)obj; + if (PyArray_NDIM(ao) > NPY_MAXDIMS_LEGACY_ITERS) { + PyErr_Format(PyExc_RuntimeError, + "this function only supports up to 32 dimensions but " + "the array has %d.", PyArray_NDIM(ao)); + return NULL; + } it = (PyArrayIterObject *)PyArray_malloc(sizeof(PyArrayIterObject)); PyObject_Init((PyObject *)it, &PyArrayIter_Type); diff --git a/numpy/_core/src/multiarray/methods.c b/numpy/_core/src/multiarray/methods.c index 71c1d2bc2d66..879d137673a3 100644 --- a/numpy/_core/src/multiarray/methods.c +++ b/numpy/_core/src/multiarray/methods.c @@ -123,7 +123,7 @@ static PyObject * array_take(PyArrayObject *self, PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int dimension = NPY_MAXDIMS; + int dimension = NPY_RAVEL_AXIS; PyObject *indices; PyArrayObject *out = NULL; NPY_CLIPMODE mode = NPY_RAISE; @@ -298,7 +298,7 @@ static PyObject * array_argmax(PyArrayObject *self, PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyArrayObject *out = NULL; npy_bool keepdims = NPY_FALSE; NPY_PREPARE_ARGPARSER; @@ -326,7 +326,7 @@ static PyObject * array_argmin(PyArrayObject *self, PyObject *const *args, Py_ssize_t len_args, PyObject *kwnames) { - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyArrayObject *out = NULL; npy_bool keepdims = NPY_FALSE; NPY_PREPARE_ARGPARSER; @@ -1185,7 +1185,7 @@ array_resize(PyArrayObject *self, PyObject *args, PyObject *kwds) static PyObject * array_repeat(PyArrayObject *self, PyObject *args, PyObject *kwds) { PyObject *repeats; - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; static char *kwlist[] = {"repeats", "axis", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|O&:repeat", kwlist, @@ -2315,7 +2315,7 @@ array_sum(PyArrayObject *self, static PyObject * array_cumsum(PyArrayObject *self, PyObject *args, PyObject *kwds) { - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyArray_Descr *dtype = NULL; PyArrayObject *out = NULL; int rtype; @@ -2344,7 +2344,7 @@ array_prod(PyArrayObject *self, static PyObject * array_cumprod(PyArrayObject *self, PyObject *args, PyObject *kwds) { - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyArray_Descr *dtype = NULL; PyArrayObject *out = NULL; int rtype; @@ -2426,7 +2426,7 @@ array_variance(PyArrayObject *self, static PyObject * array_compress(PyArrayObject *self, PyObject *args, PyObject *kwds) { - int axis = NPY_MAXDIMS; + int axis = NPY_RAVEL_AXIS; PyObject *condition; PyArrayObject *out = NULL; static char *kwlist[] = {"condition", "axis", "out", NULL}; diff --git a/numpy/_core/src/multiarray/multiarraymodule.c b/numpy/_core/src/multiarray/multiarraymodule.c index acc9f2cc0997..7429e9497ab7 100644 --- a/numpy/_core/src/multiarray/multiarraymodule.c +++ b/numpy/_core/src/multiarray/multiarraymodule.c @@ -692,7 +692,7 @@ PyArray_ConcatenateInto(PyObject *op, Py_DECREF(item); } - if (axis >= NPY_MAXDIMS) { + if (axis == NPY_RAVEL_AXIS) { ret = PyArray_ConcatenateFlattenedArrays( narrays, arrays, NPY_CORDER, ret, dtype, casting, casting_not_passed); diff --git a/numpy/_core/src/multiarray/refcount.c b/numpy/_core/src/multiarray/refcount.c index 93a7a794b7bd..3dd1228e8ebb 100644 --- a/numpy/_core/src/multiarray/refcount.c +++ b/numpy/_core/src/multiarray/refcount.c @@ -318,6 +318,13 @@ PyArray_XDECREF(PyArrayObject *mp) return 0; } if (PyArray_DESCR(mp)->type_num != NPY_OBJECT) { + if (PyArray_NDIM(mp) > NPY_MAXDIMS_LEGACY_ITERS) { + PyErr_Format(PyExc_RuntimeError, + "this function only supports up to 32 dimensions but " + "the array has %d.", PyArray_NDIM(mp)); + return -1; + } + PyArray_RawIterBaseInit(&it, mp); while(it.index < it.size) { PyArray_Item_XDECREF(it.dataptr, PyArray_DESCR(mp)); @@ -340,6 +347,13 @@ PyArray_XDECREF(PyArrayObject *mp) } } else { /* handles misaligned data too */ + if (PyArray_NDIM(mp) > NPY_MAXDIMS_LEGACY_ITERS) { + PyErr_Format(PyExc_RuntimeError, + "this function only supports up to 32 dimensions but " + "the array has %d.", PyArray_NDIM(mp)); + return -1; + } + PyArray_RawIterBaseInit(&it, mp); while(it.index < it.size) { memcpy(&temp, it.dataptr, sizeof(temp)); diff --git a/numpy/_core/src/multiarray/sequence.c b/numpy/_core/src/multiarray/sequence.c index 8db0690a1d75..7bdd64d27e5f 100644 --- a/numpy/_core/src/multiarray/sequence.c +++ b/numpy/_core/src/multiarray/sequence.c @@ -40,7 +40,7 @@ array_contains(PyArrayObject *self, PyObject *el) return -1; } - any = PyArray_Any((PyArrayObject *)res, NPY_MAXDIMS, NULL); + any = PyArray_Any((PyArrayObject *)res, NPY_RAVEL_AXIS, NULL); Py_DECREF(res); if (any == NULL) { return -1; diff --git a/numpy/_core/tests/test_conversion_utils.py b/numpy/_core/tests/test_conversion_utils.py index c42632a796a5..51676320fa0d 100644 --- a/numpy/_core/tests/test_conversion_utils.py +++ b/numpy/_core/tests/test_conversion_utils.py @@ -204,6 +204,6 @@ def test_too_large(self): self.conv(2**64) def test_too_many_dims(self): - assert self.conv([1]*32) == (1,)*32 + assert self.conv([1]*64) == (1,)*64 with pytest.raises(ValueError): - self.conv([1]*33) + self.conv([1]*65) From af55c6b7d1ba15128fe7ddd6c9d91f057faa9d28 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Nov 2023 21:04:08 +0100 Subject: [PATCH 02/10] TST: Adapt test to larger max-dims and max-args --- numpy/_core/tests/test_indexing.py | 8 ++++---- numpy/_core/tests/test_multiarray.py | 4 ++-- numpy/_core/tests/test_numeric.py | 4 ++-- numpy/_core/tests/test_overrides.py | 8 ++++---- numpy/_core/tests/test_regression.py | 4 ++-- numpy/_core/tests/test_shape_base.py | 7 ++++--- numpy/_core/tests/test_umath.py | 2 +- 7 files changed, 19 insertions(+), 18 deletions(-) diff --git a/numpy/_core/tests/test_indexing.py b/numpy/_core/tests/test_indexing.py index 204d74c4b8f1..a509df08c173 100644 --- a/numpy/_core/tests/test_indexing.py +++ b/numpy/_core/tests/test_indexing.py @@ -297,8 +297,8 @@ def test_uncontiguous_subspace_assignment(self): def test_too_many_fancy_indices_special_case(self): # Just documents behaviour, this is a small limitation. - a = np.ones((1,) * 32) # 32 is NPY_MAXDIMS - assert_raises(IndexError, a.__getitem__, (np.array([0]),) * 32) + a = np.ones((1,) * 64) # 64 is NPY_MAXDIMS + assert_raises(IndexError, a.__getitem__, (np.array([0]),) * 64) def test_scalar_array_bool(self): # NumPy bools can be used as boolean index (python ones as of yet not) @@ -552,8 +552,8 @@ def test_character_assignment(self): @pytest.mark.parametrize("index", [True, False, np.array([0])]) - @pytest.mark.parametrize("num", [32, 40]) - @pytest.mark.parametrize("original_ndim", [1, 32]) + @pytest.mark.parametrize("num", [64, 80]) + @pytest.mark.parametrize("original_ndim", [1, 64]) def test_too_many_advanced_indices(self, index, num, original_ndim): # These are limitations based on the number of arguments we can process. # For `num=32` (and all boolean cases), the result is actually define; diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index a56df4a9dc59..b41778b93715 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -744,7 +744,7 @@ def subscript(x, i): x[i] assert_raises(IndexError, subscript, a, (np.newaxis, 0)) - assert_raises(IndexError, subscript, a, (np.newaxis,)*50) + assert_raises(IndexError, subscript, a, (np.newaxis,)*70) def test_constructor(self): x = np.ndarray(()) @@ -826,7 +826,7 @@ def subscript(x, i): x[i] assert_raises(IndexError, subscript, a, (np.newaxis, 0)) - assert_raises(IndexError, subscript, a, (np.newaxis,)*50) + assert_raises(IndexError, subscript, a, (np.newaxis,)*70) def test_overlapping_assignment(self): # With positive strides diff --git a/numpy/_core/tests/test_numeric.py b/numpy/_core/tests/test_numeric.py index 7c9b3cf354ac..6fd8f5384913 100644 --- a/numpy/_core/tests/test_numeric.py +++ b/numpy/_core/tests/test_numeric.py @@ -3969,9 +3969,9 @@ def test_broadcast_single_arg(self): def test_number_of_arguments(self): arr = np.empty((5,)) - for j in range(35): + for j in range(70): arrs = [arr] * j - if j > 32: + if j > 64: assert_raises(ValueError, np.broadcast, *arrs) else: mit = np.broadcast(*arrs) diff --git a/numpy/_core/tests/test_overrides.py b/numpy/_core/tests/test_overrides.py index b9b71661db09..e03fe8bd15ab 100644 --- a/numpy/_core/tests/test_overrides.py +++ b/numpy/_core/tests/test_overrides.py @@ -134,11 +134,11 @@ class D: def test_too_many_duck_arrays(self): namespace = dict(__array_function__=_return_not_implemented) - types = [type('A' + str(i), (object,), namespace) for i in range(33)] + types = [type('A' + str(i), (object,), namespace) for i in range(65)] relevant_args = [t() for t in types] - actual = _get_implementing_args(relevant_args[:32]) - assert_equal(actual, relevant_args[:32]) + actual = _get_implementing_args(relevant_args[:64]) + assert_equal(actual, relevant_args[:64]) with assert_raises_regex(TypeError, 'distinct argument types'): _get_implementing_args(relevant_args) @@ -448,7 +448,7 @@ def func(*, like=None): def test_too_many_args(self): # Mainly a unit-test to increase coverage objs = [] - for i in range(40): + for i in range(80): class MyArr: def __array_function__(self, *args, **kwargs): return NotImplemented diff --git a/numpy/_core/tests/test_regression.py b/numpy/_core/tests/test_regression.py index f54a0def5423..85c9be247a6b 100644 --- a/numpy/_core/tests/test_regression.py +++ b/numpy/_core/tests/test_regression.py @@ -1230,7 +1230,7 @@ def test_for_zero_length_in_choose(self): def test_array_ndmin_overflow(self): "Ticket #947." - assert_raises(ValueError, lambda: np.array([1], ndmin=33)) + assert_raises(ValueError, lambda: np.array([1], ndmin=65)) def test_void_scalar_with_titles(self): # No ticket @@ -2210,7 +2210,7 @@ def test_frompyfunc_many_args(self): def passer(*args): pass - assert_raises(ValueError, np.frompyfunc, passer, 32, 1) + assert_raises(ValueError, np.frompyfunc, passer, 64, 1) def test_repeat_broadcasting(self): # gh-5743 diff --git a/numpy/_core/tests/test_shape_base.py b/numpy/_core/tests/test_shape_base.py index 46931e94215e..5b9de37f5f60 100644 --- a/numpy/_core/tests/test_shape_base.py +++ b/numpy/_core/tests/test_shape_base.py @@ -302,9 +302,10 @@ def test_large_concatenate_axis_None(self): r = np.concatenate(x, None) assert_array_equal(x, r) - # This should probably be deprecated: - r = np.concatenate(x, 100) # axis is >= MAXDIMS - assert_array_equal(x, r) + # Once upon a time, this was the same as `axis=None` now it fails + # (with an unspecified error, as multiple things are wrong here) + with pytest.raises(ValueError): + np.concatenate(x, 100) def test_concatenate(self): # Test concatenate function diff --git a/numpy/_core/tests/test_umath.py b/numpy/_core/tests/test_umath.py index 7b0fbd64b6b9..73fa14c26501 100644 --- a/numpy/_core/tests/test_umath.py +++ b/numpy/_core/tests/test_umath.py @@ -4666,7 +4666,7 @@ def __array_finalize__(self, obj): assert type(np.add.outer([1, 2], arr)) is cls def test_outer_exceeds_maxdims(): - deep = np.ones((1,) * 17) + deep = np.ones((1,) * 33) with assert_raises(ValueError): np.add.outer(deep, deep) From bbc40836309c8ec49bce25582ac23214b7e733ff Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Tue, 14 Nov 2023 22:31:40 +0100 Subject: [PATCH 03/10] TST: Add a few tests for paths that are for now limited to 32 dims --- numpy/_core/tests/test_multiarray.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index b41778b93715..47d9c9d318cf 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -5828,6 +5828,15 @@ def test_index_getset(self): # If the type was incorrect, this would show up on big-endian machines assert it.index == it.base.size + def test_maxdims(self): + # The flat iterator and thus attribute is currently unfortunately + # limited to only 32 dimensions (after bumping it to 64 for 2.0) + a = np.ones((1,) * 64) + + with pytest.raises(RuntimeError, + match=".*32 dimensions but the array has 64"): + a.flat + class TestResize: @@ -7406,6 +7415,22 @@ def test_output_dtype(self, ops): expected_dt = np.result_type(*ops) assert(np.choose([0], ops).dtype == expected_dt) + def test_dimension_and_args_limit(self): + # Maxdims for the legacy iterator is 32, but the maximum number + # of arguments is actually larger (a itself also counts here) + a = np.ones((1,) * 32, dtype=np.intp) + res = a.choose([0, a] + [2] * 61) + with pytest.raises(ValueError, + match="Need at least 0 and at most 64 array objects"): + a.choose([0, a] + [2] * 62) + + assert_array_equal(res, a) + # Choose is unfortunately limited to 32 dims as of NumPy 2.0 + a = np.ones((1,) * 60, dtype=np.intp) + with pytest.raises(RuntimeError, + match=".*32 dimensions but the array has 60"): + a.choose([a, a]) + class TestRepeat: def setup_method(self): From 6a878e38fc5744b3827d751c8371e44558b6a36b Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 10:55:31 +0100 Subject: [PATCH 04/10] API: Add 1.x support for ``PyArray_RUNTIME_VERSION`` to the compat header --- numpy/_core/include/numpy/npy_2_compat.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/numpy/_core/include/numpy/npy_2_compat.h b/numpy/_core/include/numpy/npy_2_compat.h index fcda1c69c674..2350a74235a3 100644 --- a/numpy/_core/include/numpy/npy_2_compat.h +++ b/numpy/_core/include/numpy/npy_2_compat.h @@ -30,6 +30,20 @@ #ifndef NUMPY_CORE_INCLUDE_NUMPY_NPY_2_COMPAT_H_ #define NUMPY_CORE_INCLUDE_NUMPY_NPY_2_COMPAT_H_ +/* + * Allow users to use `PyArray_RUNTIME_VERSION` when vendoring the file for + * compilation with NumPy 1.x. + * Simply do not define when compiling with 2.x. It must be defined later + * as it is set during `import_array()`. + */ +#if !defined(PyArray_RUNTIME_VERSION) && NPY_ABI_VERSION < 0x02000000 + /* + * If we are compiling with NumPy 1.x, PyArray_RUNTIME_VERSION so we + * pretend the `PyArray_RUNTIME_VERSION` is `NPY_FEATURE_VERSION`. + */ + #define PyArray_RUNTIME_VERSION NPY_FEATURE_VERSION +#endif + /* * New macros for accessing real and complex part of a complex number can be * found in "npy_2_complexcompat.h". From 3270231a4c333997066f35815b1ee05b12adccf0 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 10:58:48 +0100 Subject: [PATCH 05/10] DOC: Add documentation for maxdim increase (and related) changes --- doc/release/upcoming_changes/25149.c_api.rst | 10 ++++++ doc/release/upcoming_changes/25149.change.rst | 8 +++++ doc/source/numpy_2_0_migration_guide.rst | 22 +++++++++++++ doc/source/reference/c-api/array.rst | 32 +++++++++++++++---- 4 files changed, 66 insertions(+), 6 deletions(-) create mode 100644 doc/release/upcoming_changes/25149.c_api.rst create mode 100644 doc/release/upcoming_changes/25149.change.rst diff --git a/doc/release/upcoming_changes/25149.c_api.rst b/doc/release/upcoming_changes/25149.c_api.rst new file mode 100644 index 000000000000..1bc6fe12c257 --- /dev/null +++ b/doc/release/upcoming_changes/25149.c_api.rst @@ -0,0 +1,10 @@ +Lager ``NPY_MAXDIMS`` and ``NPY_MAXARGS``, ``NPY_RAVEL_AXIS`` introduced +------------------------------------------------------------------------ + +``NPY_MAXDIMS`` is now 64, you may want to review its use. This is usually +used in stack allocation, where the increase should be safe. +However, we do encourage generally to remove any use of ``NPY_MAXDIMS`` and +``NPY_MAXARGS`` to eventually allow removing the constraint completely. +In some cases, ``NPY_MAXDIMS`` was passed (and returned) to mean ``axis=None`` +these must be replaced with ``NPY_RAVEL_AXIS``. +See also :ref:`migration_maxdims`. \ No newline at end of file diff --git a/doc/release/upcoming_changes/25149.change.rst b/doc/release/upcoming_changes/25149.change.rst new file mode 100644 index 000000000000..2965877036b9 --- /dev/null +++ b/doc/release/upcoming_changes/25149.change.rst @@ -0,0 +1,8 @@ +Out-of-bound axis not the same as ``axis=None`` +----------------------------------------------- +In some cases ``axis=32`` or for concatenate any large value +was the same as ``axis=None``. +Except for ``concatenate`` this was deprecate. +Any out of bound axis value will now error, make sure to use +``axis=None``. + diff --git a/doc/source/numpy_2_0_migration_guide.rst b/doc/source/numpy_2_0_migration_guide.rst index 104d1655a2b9..9bfb619dfa30 100644 --- a/doc/source/numpy_2_0_migration_guide.rst +++ b/doc/source/numpy_2_0_migration_guide.rst @@ -41,8 +41,30 @@ Some are defined in ``numpy/_core/include/numpy/npy_2_compat.h`` (for example ``NPY_DEFAULT_INT``) which can be vendored in full or part to have the definitions available when compiling against NumPy 1.x. +If necessary, ``PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION`` can be +used to explicitly implement different behavior on NumPy 1.x and 2.0. +(The compat header defines it in a way compatible with such use.) + Please let us know if you require additional workarounds here. +.. _migration_maxdims: + +Increased maximum number of dimensions +-------------------------------------- +The maximum number of dimensions (and arguments) was increased to 64, this +affects the ``NPY_MAXDIMS`` and ``NPY_MAXARGS`` macros. +It may be good to review their use, and we generally encourage you to +not use this macros (especially ``NPY_MAXARGS``), so that a future version of +NumPy can remove this limitation on the number of dimensions. + +``NPY_MAXDIMS`` was also used to signal ``axis=None`` in the C-API, including +the ``PyArray_AxisConverter``. +If you run into this problem, you will see ``-2147483648`` +(the minimum integer value) or ``64`` being used as an invalid axis. +Wherever you do, you must replace ``NPY_MAXDIMS`` with ``NPY_RAVEL_AXIS``. +This is defined in the ``npy_2_compat.h`` header and runtime dependent. + + Namespace changes ================= diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 18be77512b65..b700fc736c87 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -1949,7 +1949,7 @@ Calculation .. tip:: - Pass in :c:data:`NPY_MAXDIMS` for axis in order to achieve the same + Pass in :c:data:`NPY_RAVEL_AXIS` for axis in order to achieve the same effect that is obtained by passing in ``axis=None`` in Python (treating the array as a 1-d array). @@ -2968,7 +2968,7 @@ to. Convert a Python object, *obj*, representing an axis argument to the proper value for passing to the functions that take an integer axis. Specifically, if *obj* is None, *axis* is set to - :c:data:`NPY_MAXDIMS` which is interpreted correctly by the C-API + :c:data:`NPY_RAVEL_AXIS` which is interpreted correctly by the C-API functions that take axis arguments. .. c:function:: int PyArray_BoolConverter(PyObject* obj, npy_bool* value) @@ -3411,14 +3411,26 @@ Other constants .. c:macro:: NPY_MAXDIMS + The maximum number of dimensions that may be used by NumPy. + This is set to 64 and was 32 before NumPy 2. + .. note:: - NumPy used to have a maximum number of dimensions set to 32 and now 64. - This was/is mostly useful for stack allocations. We now ask you to - define (and test) such a limit in your own project if needed. + We encourage you to avoid ``NPY_MAXDIMS``. A future version of NumPy + may wish to remove any dimension limitation (and thus the constant). + The limitation was created mainly a simplification for the liberal use + of small stack allocations as scratch space. + + If your algorithm has a reasonable maximum number of dimension you + could check and use that locally. .. c:macro:: NPY_MAXARGS - The maximum number of array arguments that can be used in functions. + The maximum number of array arguments that can be used in in some + functions. This used to be 32 before NumPy 2 and is now 64. + + .. note:: + You should never use this. We may remove it in future versions of + NumPy. .. c:macro:: NPY_FALSE @@ -3438,6 +3450,14 @@ Other constants The return value of successful converter functions which are called using the "O&" syntax in :c:func:`PyArg_ParseTuple`-like functions. +.. c:macro:: NPY_RAVEL_AXIS + + Some NumPy functions (mainly the C-entrypoints for Python functions) + have an ``axis`` argument. This macro may be passed for ``axis=None``. + + .. note:: + This macro is a NumPy version dependent at runtime. + Miscellaneous Macros ~~~~~~~~~~~~~~~~~~~~ From d6840f54bf8e390686045537a0a9f08be2ecf47e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 16:06:48 +0100 Subject: [PATCH 06/10] TST: "fix" failing memoryview with too many dimensions test Memoryview actually also only supports 65 dimensions currently, so skip the test when the memoryview creation fails. If it were to work, the test should kick in and work fine. --- numpy/_core/tests/test_multiarray.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index 47d9c9d318cf..0c72d770a064 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -8204,19 +8204,22 @@ def make_ctype(shape, scalar_type): t = dim * t return t - # construct a memoryview with 33 dimensions - c_u8_33d = make_ctype((1,)*33, ctypes.c_uint8) - m = memoryview(c_u8_33d()) - assert_equal(m.ndim, 33) + # Try constructing a memory with too many dimensions: + c_u8_65d = make_ctype((1,)*65, ctypes.c_uint8) + try: + m = memoryview(c_u8_65d()) + except ValueError: + pytest.skip("memoryview doesn't support more dimensions") - assert_raises_regex( - RuntimeError, "ndim", - np.array, m) + assert_equal(m.ndim, 65) + + with pytest.raises(RuntimeError, match=".*ndim"): + np.array(m) # The above seems to create some deep cycles, clean them up for # easier reference count debugging: - del c_u8_33d, m - for i in range(33): + del c_u8_65d, m + for i in range(65): if gc.collect() == 0: break From a5a250f16885d5c3ad48e711bd3925c753371ddd Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 16:40:01 +0100 Subject: [PATCH 07/10] TST: Make linter happy, it was right. --- numpy/_core/tests/test_multiarray.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/numpy/_core/tests/test_multiarray.py b/numpy/_core/tests/test_multiarray.py index 0c72d770a064..d8dd55a1acdf 100644 --- a/numpy/_core/tests/test_multiarray.py +++ b/numpy/_core/tests/test_multiarray.py @@ -7422,7 +7422,7 @@ def test_dimension_and_args_limit(self): res = a.choose([0, a] + [2] * 61) with pytest.raises(ValueError, match="Need at least 0 and at most 64 array objects"): - a.choose([0, a] + [2] * 62) + a.choose([0, a] + [2] * 62) assert_array_equal(res, a) # Choose is unfortunately limited to 32 dims as of NumPy 2.0 From 8dddc54a764c757a21d0259b5a1d937897ae9b50 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Wed, 15 Nov 2023 16:44:07 +0100 Subject: [PATCH 08/10] MAINT: Cython restore NPY_MAXDIMS and add NPY_RAVEL_AXIS --- numpy/__init__.cython-30.pxd | 4 ++++ numpy/__init__.pxd | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/numpy/__init__.cython-30.pxd b/numpy/__init__.cython-30.pxd index 6b3d57a31f01..865a94c1f0a7 100644 --- a/numpy/__init__.cython-30.pxd +++ b/numpy/__init__.cython-30.pxd @@ -186,6 +186,10 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_UPDATE_ALL + cdef enum: + NPY_MAXDIMS # 64 on NumPy 2.x and 32 on NumPy 1.x + NPY_RAVEL_AXIS # Used for functions like PyArray_Mean + ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) ctypedef struct PyArray_ArrayDescr: diff --git a/numpy/__init__.pxd b/numpy/__init__.pxd index 17a29f3979aa..12ed0924e135 100644 --- a/numpy/__init__.pxd +++ b/numpy/__init__.pxd @@ -181,6 +181,10 @@ cdef extern from "numpy/arrayobject.h": NPY_ARRAY_UPDATE_ALL + cdef enum: + NPY_MAXDIMS # 64 on NumPy 2.x and 32 on NumPy 1.x + NPY_RAVEL_AXIS # Used for functions like PyArray_Mean + ctypedef void (*PyArray_VectorUnaryFunc)(void *, void *, npy_intp, void *, void *) ctypedef struct PyArray_ArrayDescr: From 56b792ce6d84984db8709b4a57cd794a6d02b34e Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 24 Nov 2023 12:20:03 +0100 Subject: [PATCH 09/10] Apply suggestions from code review Co-authored-by: Nathan Goldbaum --- doc/release/upcoming_changes/25149.c_api.rst | 2 +- doc/source/numpy_2_0_migration_guide.rst | 2 +- doc/source/reference/c-api/array.rst | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/doc/release/upcoming_changes/25149.c_api.rst b/doc/release/upcoming_changes/25149.c_api.rst index 1bc6fe12c257..9e0729be9ad3 100644 --- a/doc/release/upcoming_changes/25149.c_api.rst +++ b/doc/release/upcoming_changes/25149.c_api.rst @@ -2,7 +2,7 @@ Lager ``NPY_MAXDIMS`` and ``NPY_MAXARGS``, ``NPY_RAVEL_AXIS`` introduced ------------------------------------------------------------------------ ``NPY_MAXDIMS`` is now 64, you may want to review its use. This is usually -used in stack allocation, where the increase should be safe. +used in a stack allocation, where the increase should be safe. However, we do encourage generally to remove any use of ``NPY_MAXDIMS`` and ``NPY_MAXARGS`` to eventually allow removing the constraint completely. In some cases, ``NPY_MAXDIMS`` was passed (and returned) to mean ``axis=None`` diff --git a/doc/source/numpy_2_0_migration_guide.rst b/doc/source/numpy_2_0_migration_guide.rst index 9bfb619dfa30..994ad30b8b98 100644 --- a/doc/source/numpy_2_0_migration_guide.rst +++ b/doc/source/numpy_2_0_migration_guide.rst @@ -54,7 +54,7 @@ Increased maximum number of dimensions The maximum number of dimensions (and arguments) was increased to 64, this affects the ``NPY_MAXDIMS`` and ``NPY_MAXARGS`` macros. It may be good to review their use, and we generally encourage you to -not use this macros (especially ``NPY_MAXARGS``), so that a future version of +not use these macros (especially ``NPY_MAXARGS``), so that a future version of NumPy can remove this limitation on the number of dimensions. ``NPY_MAXDIMS`` was also used to signal ``axis=None`` in the C-API, including diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index b700fc736c87..4913813c00e8 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3417,15 +3417,15 @@ Other constants .. note:: We encourage you to avoid ``NPY_MAXDIMS``. A future version of NumPy may wish to remove any dimension limitation (and thus the constant). - The limitation was created mainly a simplification for the liberal use - of small stack allocations as scratch space. + The limitation was created so that NumPy can use stack allocations + internally for scratch space. If your algorithm has a reasonable maximum number of dimension you could check and use that locally. .. c:macro:: NPY_MAXARGS - The maximum number of array arguments that can be used in in some + The maximum number of array arguments that can be used in some functions. This used to be 32 before NumPy 2 and is now 64. .. note:: @@ -3456,7 +3456,7 @@ Other constants have an ``axis`` argument. This macro may be passed for ``axis=None``. .. note:: - This macro is a NumPy version dependent at runtime. + This macro is NumPy version dependent at runtime. Miscellaneous Macros From 0498d8cf18e1d061d17622fac680f901e5f15e36 Mon Sep 17 00:00:00 2001 From: Sebastian Berg Date: Fri, 24 Nov 2023 13:33:59 +0100 Subject: [PATCH 10/10] Address review comments --- doc/release/upcoming_changes/25149.c_api.rst | 6 +++--- doc/release/upcoming_changes/25149.c_api_removal.rst | 2 ++ doc/source/numpy_2_0_migration_guide.rst | 10 ++++++---- doc/source/reference/c-api/array.rst | 4 +++- doc/source/reference/c-api/types-and-structures.rst | 9 +++++---- 5 files changed, 19 insertions(+), 12 deletions(-) create mode 100644 doc/release/upcoming_changes/25149.c_api_removal.rst diff --git a/doc/release/upcoming_changes/25149.c_api.rst b/doc/release/upcoming_changes/25149.c_api.rst index 9e0729be9ad3..27907b5a90fb 100644 --- a/doc/release/upcoming_changes/25149.c_api.rst +++ b/doc/release/upcoming_changes/25149.c_api.rst @@ -5,6 +5,6 @@ Lager ``NPY_MAXDIMS`` and ``NPY_MAXARGS``, ``NPY_RAVEL_AXIS`` introduced used in a stack allocation, where the increase should be safe. However, we do encourage generally to remove any use of ``NPY_MAXDIMS`` and ``NPY_MAXARGS`` to eventually allow removing the constraint completely. -In some cases, ``NPY_MAXDIMS`` was passed (and returned) to mean ``axis=None`` -these must be replaced with ``NPY_RAVEL_AXIS``. -See also :ref:`migration_maxdims`. \ No newline at end of file +For the conversion helper and C-API functions mirrowing Python ones such as +``tale``, ``NPY_MAXDIMS`` was used to mean ``axis=None`` these must be +replaced with ``NPY_RAVEL_AXIS``. See also :ref:`migration_maxdims`. \ No newline at end of file diff --git a/doc/release/upcoming_changes/25149.c_api_removal.rst b/doc/release/upcoming_changes/25149.c_api_removal.rst new file mode 100644 index 000000000000..48e1d560536f --- /dev/null +++ b/doc/release/upcoming_changes/25149.c_api_removal.rst @@ -0,0 +1,2 @@ +* ``NPY_MAX_ELSIZE`` macro has been removed as it only ever reflected + builtin numeric types and served no internal purpose. diff --git a/doc/source/numpy_2_0_migration_guide.rst b/doc/source/numpy_2_0_migration_guide.rst index 994ad30b8b98..0253d0753345 100644 --- a/doc/source/numpy_2_0_migration_guide.rst +++ b/doc/source/numpy_2_0_migration_guide.rst @@ -59,10 +59,12 @@ NumPy can remove this limitation on the number of dimensions. ``NPY_MAXDIMS`` was also used to signal ``axis=None`` in the C-API, including the ``PyArray_AxisConverter``. -If you run into this problem, you will see ``-2147483648`` -(the minimum integer value) or ``64`` being used as an invalid axis. -Wherever you do, you must replace ``NPY_MAXDIMS`` with ``NPY_RAVEL_AXIS``. -This is defined in the ``npy_2_compat.h`` header and runtime dependent. +The latter will return ``-2147483648`` as an axis (the smallest integer value). +Other functions may error with +``AxisError: axis 64 is out of bounds for array of dimension`` in which +case you need to pass ``NPY_RAVEL_AXIS`` instead of ``NPY_MAXDIMS``. +``NPY_RAVEL_AXIS`` is defined in the ``npy_2_compat.h`` header and runtime +dependent (mapping to 32 on NumPy 1.x and ``-2147483648`` on NumPy 2.x). Namespace changes diff --git a/doc/source/reference/c-api/array.rst b/doc/source/reference/c-api/array.rst index 4913813c00e8..9c51dc53d95b 100644 --- a/doc/source/reference/c-api/array.rst +++ b/doc/source/reference/c-api/array.rst @@ -3456,7 +3456,9 @@ Other constants have an ``axis`` argument. This macro may be passed for ``axis=None``. .. note:: - This macro is NumPy version dependent at runtime. + This macro is NumPy version dependent at runtime. The value is now + the minimum integer. However, on NumPy 1.x ``NPY_MAXDIMS`` was used + (at the time set to 32). Miscellaneous Macros diff --git a/doc/source/reference/c-api/types-and-structures.rst b/doc/source/reference/c-api/types-and-structures.rst index b6b00bc9f19c..54a62b4e10f2 100644 --- a/doc/source/reference/c-api/types-and-structures.rst +++ b/doc/source/reference/c-api/types-and-structures.rst @@ -120,10 +120,11 @@ PyArray_Type and PyArrayObject array. Such arrays have undefined dimensions and strides and cannot be accessed. Macro :c:data:`PyArray_NDIM` defined in ``ndarraytypes.h`` points to this data member. - Although most operations may be limited in dimensionality, we do not - advertise a maximum dimension. Anyone explicitly relying on one - must check for it. Before NumPy 2.0, NumPy used 32 dimensions at most - after it, the limit is currently 64. + ``NPY_MAXDIMS`` is defined as a compile time constant limiting the + number of dimensions. This number is 64 since NumPy 2 and was 32 + before. However, we may wish to remove this limitations in the future + so that it is best to explicitly check dimensionality for code + that relies on such an upper bound. .. c:member:: npy_intp *dimensions