Merge pull request numpy#25797 from seberg/str-typenum

MAINT: Move ``NPY_VSTRING`` and make ``NPY_NTYPES NPY_TYPES_LEGACY``

doc/release/upcoming_changes/25347.c_api.rst

@@ -2,8 +2,6 @@
   exposed. This includes functions for acquiring and releasing mutexes locking
   access to the string data as well as packing and unpacking UTF-8 bytestreams
   from array entries.
-* ``NPY_NTYPES`` is now version-dependent to accomadate the availability of a
-  new NumPy built-in DType. It is now defined in ``npy2_compat.h``.
-* ``NPY_NTYPES_LEGACY`` is now defined in ``ndarraytypes.h``, use this value
-  if you do not wish to update code to support dtypes written using the new
-  DType API that may not function in the same way as legacy dtypes.
+* ``NPY_NTYPES`` has been renamed to ``NPY_NTYPES_LEGACY`` as it does not
+  include new NumPy built-in DTypes. In particular the new string DType
+  will likely not work correctly with code that handles legacy DTypes.

doc/source/reference/c-api/dtype.rst

@@ -154,6 +154,9 @@ Enumerated types
         the array buffer. Use the C API for working with numpy variable-width
         static strings to access the string data in each array entry.
 
+        .. note::
+            This DType is new-style and is not included in ``NPY_NTYPES_LEGACY``.
+
     .. c:enumerator:: NPY_OBJECT
 
         The enumeration value for references to arbitrary Python objects.
@@ -190,27 +193,31 @@ Enumerated types
 
 Other useful related constants are
 
-.. c:macro:: NPY_NTYPES
-
-    The total number of built-in NumPy types. The enumeration covers
-    the range from 0 to NPY_NTYPES-1.
-
 .. c:macro:: NPY_NTYPES_LEGACY
 
     The number of built-in NumPy types written using the legacy DType
     system. New NumPy dtypes will be written using the new DType API and may not
     function in the same manner as legacy DTypes. Use this macro if you want to
     handle legacy DTypes using different code paths or if you do not want to
-    update code that uses ``NPY_NTYPES`` and does not work correctly with new
+    update code that uses ``NPY_NTYPES_LEGACY`` and does not work correctly with new
     DTypes.
 
+    .. note::
+        Newly added DTypes such as ``NPY_VSTRING`` will not be counted
+        in ``NPY_NTYPES_LEGACY``.
+
 .. c:macro:: NPY_NOTYPE
 
     A signal value guaranteed not to be a valid type enumeration number.
 
 .. c:macro:: NPY_USERDEF
 
-    The start of type numbers used for Custom Data types.
+    The start of type numbers used for legacy Custom Data types.
+    New-style user DTypes currently are currently *not* assigned a type-number.
+
+    .. note::
+        The total number of user dtypes is limited to below ``NPY_VSTRING``.
+        Higher numbers are reserved to future new-style DType use.
 
 The various character codes indicating certain types are also part of
 an enumerated list. References to type characters (should they be

doc/source/reference/c-api/types-and-structures.rst

@@ -477,7 +477,7 @@ PyArrayDescr_Type and PyArray_Descr
     .. code-block:: c
 
        typedef struct {
-           PyArray_VectorUnaryFunc *cast[NPY_NTYPES];
+           PyArray_VectorUnaryFunc *cast[NPY_NTYPES_LEGACY];
            PyArray_GetItemFunc *getitem;
            PyArray_SetItemFunc *setitem;
            PyArray_CopySwapNFunc *copyswapn;

numpy/__init__.cython-30.pxd

@@ -53,7 +53,7 @@ cdef extern from "numpy/arrayobject.h":
         NPY_VOID
         NPY_DATETIME
         NPY_TIMEDELTA
-        NPY_NTYPES
+        NPY_NTYPES_LEGACY
         NPY_NOTYPE
 
         NPY_INT8

numpy/__init__.pxd

@@ -48,7 +48,7 @@ cdef extern from "numpy/arrayobject.h":
         NPY_VOID
         NPY_DATETIME
         NPY_TIMEDELTA
-        NPY_NTYPES
+        NPY_NTYPES_LEGACY
         NPY_NOTYPE
 
         NPY_INT8

numpy/_core/include/numpy/ndarraytypes.h

@@ -63,26 +63,27 @@ enum NPY_TYPES {    NPY_BOOL=0,
 
                     NPY_CHAR, /* Deprecated, will raise if used */
 
-                    /*
-                     * New types added after NumPy 2.0
-                     */
-                    NPY_VSTRING,
-
-                    /* NPY_NTYPES is version-dependent and defined in
-                       npy_2_compat.h */
+                    /* The number of *legacy* dtypes */
+                    NPY_NTYPES_LEGACY=24,
 
                     /* assign a high value to avoid changing this in the
                        future when new dtypes are added */
-                    NPY_NOTYPE=64,
+                    NPY_NOTYPE=25,
 
                     NPY_USERDEF=256,  /* leave room for characters */
 
                     /* The number of types not including the new 1.6 types */
                     NPY_NTYPES_ABI_COMPATIBLE=21,
+
+                    /*
+                     * New DTypes which do not share the legacy layout
+                     * (added after NumPy 2.0).  VSTRING is the first of these
+                     * we may open up a block for user-defined dtypes in the
+                     * future.
+                     */
+                    NPY_VSTRING=2056,
 };
 
-/* The number of legacy old-style dtypes */
-#define NPY_NTYPES_LEGACY 24
 
 /* basetype array priority */
 #define NPY_PRIORITY 0.0

numpy/_core/include/numpy/npy_2_compat.h

@@ -66,29 +66,28 @@
  * A constant indicating the maximum number dimensions allowed when creating
  * an ndarray.
  *
- * NPY_NTYPES
+ * NPY_NTYPES_LEGACY
  *
  * The number of built-in NumPy dtypes.
  */
 #if NPY_FEATURE_VERSION >= NPY_2_0_API_VERSION
     #define NPY_DEFAULT_INT NPY_INTP
     #define NPY_RAVEL_AXIS NPY_MIN_INT
     #define NPY_MAXARGS 64
-    #define NPY_NTYPES 24
 #elif NPY_ABI_VERSION < 0x02000000
     #define NPY_DEFAULT_INT NPY_LONG
     #define NPY_RAVEL_AXIS 32
     #define NPY_MAXARGS 32
-    #define NPY_NTYPES 25
+
+    /* Renamed in 2.x */
+    #define NPY_NTYPES NPY_NTYPES_LEGACY
 #else
     #define NPY_DEFAULT_INT  \
         (PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION ? NPY_INTP : NPY_LONG)
     #define NPY_RAVEL_AXIS  \
         (PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION ? -1 : 32)
     #define NPY_MAXARGS  \
         (PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION ? 64 : 32)
-    #define NPY_NTYPES  \
-        (PyArray_RUNTIME_VERSION >= NPY_2_0_API_VERSION ? 25 : 24)
 #endif
 
 

numpy/_core/src/multiarray/abstractdtypes.c

@@ -151,7 +151,7 @@ initialize_and_map_pytypes_to_dtypes()
 static PyArray_DTypeMeta *
 int_common_dtype(PyArray_DTypeMeta *NPY_UNUSED(cls), PyArray_DTypeMeta *other)
 {
-    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES) {
+    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES_LEGACY) {
         if (other->type_num == NPY_BOOL) {
             /* Use the default integer for bools: */
             return NPY_DT_NewRef(&PyArray_IntpDType);
@@ -199,7 +199,7 @@ int_common_dtype(PyArray_DTypeMeta *NPY_UNUSED(cls), PyArray_DTypeMeta *other)
 static PyArray_DTypeMeta *
 float_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 {
-    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES) {
+    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES_LEGACY) {
         if (other->type_num == NPY_BOOL || PyTypeNum_ISINTEGER(other->type_num)) {
             /* Use the default integer for bools and ints: */
             return NPY_DT_NewRef(&PyArray_DoubleDType);
@@ -235,7 +235,7 @@ float_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 static PyArray_DTypeMeta *
 complex_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 {
-    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES) {
+    if (NPY_DT_is_legacy(other) && other->type_num < NPY_NTYPES_LEGACY) {
         if (other->type_num == NPY_BOOL ||
                 PyTypeNum_ISINTEGER(other->type_num)) {
             /* Use the default integer for bools and ints: */

numpy/_core/src/multiarray/arraytypes.c.src

@@ -2314,7 +2314,7 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride,
                 !PyDataType_HASFIELDS(descr->subarray->base) &&
                 !PyDataType_HASSUBARRAY(descr->subarray->base) &&
                 !PyDataType_REFCHK(descr->subarray->base) &&
-                (descr->subarray->base->type_num < NPY_NTYPES));
+                (descr->subarray->base->type_num < NPY_NTYPES_LEGACY));
 
         if (can_optimize_subarray) {
             _basic_copyn(dst, dstride, src, sstride, n, descr->elsize);
@@ -2399,7 +2399,7 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr)
                 !PyDataType_HASFIELDS(descr->subarray->base) &&
                 !PyDataType_HASSUBARRAY(descr->subarray->base) &&
                 !PyDataType_REFCHK(descr->subarray->base) &&
-                (descr->subarray->base->type_num < NPY_NTYPES));
+                (descr->subarray->base->type_num < NPY_NTYPES_LEGACY));
 
         if (can_optimize_subarray) {
             _basic_copy(dst, src, descr->elsize);
@@ -4231,8 +4231,10 @@ NPY_NO_EXPORT PyArray_Descr @from@_Descr = {
 
 /**end repeat**/
 
-#define _MAX_LETTER 128
-static char _letter_to_num[_MAX_LETTER];
+/* The smallest type number is ?, the largest bounded by 'z'. */
+#define _MAX_LETTER ('z' + 1)
+static npy_int16 _letter_to_num[_MAX_LETTER - '?'];
+#define LETTER_TO_NUM(letter) _letter_to_num[letter - '?']
 
 static PyArray_Descr *_builtin_descrs[] = {
     &BOOL_Descr,
@@ -4283,7 +4285,7 @@ PyArray_DescrFromType(int type)
         ret = (PyArray_Descr *)new_stringdtype_instance(NULL, 1, NULL);
     }
     // builtin legacy dtypes
-    else if (type < NPY_NTYPES) {
+    else if (type < NPY_NTYPES_LEGACY) {
         ret = _builtin_descrs[type];
     }
     else if (type == NPY_NOTYPE) {
@@ -4311,11 +4313,11 @@ PyArray_DescrFromType(int type)
         ret = userdescrs[type - NPY_USERDEF];
     }
     else {
-        int num = NPY_NTYPES;
-        if (type < _MAX_LETTER) {
-            num = (int) _letter_to_num[type];
+        int num = NPY_NTYPES_LEGACY;
+        if (type >= '?' && type < _MAX_LETTER) {
+            num = (int) LETTER_TO_NUM(type);
         }
-        if (num >= NPY_NTYPES) {
+        if (num < 0 || num >= NPY_NTYPES_LEGACY) {
             ret = NULL;
         }
         else {
@@ -4469,8 +4471,8 @@ set_typeinfo(PyObject *dict)
         return -1;
     }
 
-    for (i = 0; i < _MAX_LETTER; i++) {
-        _letter_to_num[i] = NPY_NTYPES;
+    for (i = '?'; i < _MAX_LETTER; i++) {
+        LETTER_TO_NUM(i) = -1;
     }
 
     /**begin repeat
@@ -4484,23 +4486,23 @@ set_typeinfo(PyObject *dict)
      *         DATETIME,TIMEDELTA#
      */
 
-    _letter_to_num[NPY_@name@LTR] = NPY_@name@;
+    LETTER_TO_NUM(NPY_@name@LTR) = NPY_@name@;
 
     /**end repeat**/
-    _letter_to_num['n'] = NPY_INTP;
-    _letter_to_num['N'] = NPY_UINTP;
+    LETTER_TO_NUM('n') = NPY_INTP;
+    LETTER_TO_NUM('N') = NPY_UINTP;
 
 #if NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_INTP
-    _letter_to_num['p'] = NPY_INTP;
-    _letter_to_num['P'] = NPY_UINTP;
+    LETTER_TO_NUM('p') = NPY_INTP;
+    LETTER_TO_NUM('P') = NPY_UINTP;
 #elif NPY_SIZEOF_PY_INTPTR_T == NPY_SIZEOF_LONGLONG
-    _letter_to_num['p'] = NPY_LONGLONG;
-    _letter_to_num['P'] = NPY_ULONGLONG;
+    LETTER_TO_NUM('p') = NPY_LONGLONG;
+    LETTER_TO_NUM('P') = NPY_ULONGLONG;
 #else
     #error "Did not find correct pointer sized integer."
 #endif
 
-    _letter_to_num['T'] = NPY_VSTRING;
+    LETTER_TO_NUM('T') = NPY_VSTRING;
 
     /**begin repeat
       * #name = BOOL,

numpy/_core/src/multiarray/can_cast_table.h

@@ -63,7 +63,7 @@
      FITS(FROM, FLOAT), FITS(FROM, DOUBLE), FITS(FROM, LONGDOUBLE),  \
      1, 1, 1, 1, 0, 0, 0}
 
-static const npy_bool _npy_can_cast_safely_table[NPY_NTYPES][NPY_NTYPES] = {
+static const npy_bool _npy_can_cast_safely_table[NPY_NTYPES_LEGACY][NPY_NTYPES_LEGACY] = {
         /* Bool safely casts to anything except datetime (has no zero) */
         {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
          1, 1, 1, 1, 1, 1,

numpy/_core/src/multiarray/convert_datatype.c

@@ -190,7 +190,7 @@ PyArray_GetCastingImpl(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to)
     else if (!NPY_DT_is_legacy(from) || !NPY_DT_is_legacy(to)) {
         Py_RETURN_NONE;
     }
-    else if (from->type_num < NPY_NTYPES && to->type_num < NPY_NTYPES) {
+    else if (from->type_num < NPY_NTYPES_LEGACY && to->type_num < NPY_NTYPES_LEGACY) {
         /* All builtin dtypes have their casts explicitly defined. */
         PyErr_Format(PyExc_RuntimeError,
                 "builtin cast from %S to %S not found, this should not "
@@ -1009,7 +1009,7 @@ promote_types(PyArray_Descr *type1, PyArray_Descr *type2,
         int type_num2 = type2->type_num;
         int ret_type_num;
 
-        if (type_num2 < NPY_NTYPES && !(PyTypeNum_ISBOOL(type_num2) ||
+        if (type_num2 < NPY_NTYPES_LEGACY && !(PyTypeNum_ISBOOL(type_num2) ||
                                         PyTypeNum_ISUNSIGNED(type_num2))) {
             /* Convert to the equivalent-sized signed integer */
             type_num1 = type_num_unsigned_to_signed(type_num1);
@@ -1028,7 +1028,7 @@ promote_types(PyArray_Descr *type1, PyArray_Descr *type2,
         int type_num2 = type2->type_num;
         int ret_type_num;
 
-        if (type_num1 < NPY_NTYPES && !(PyTypeNum_ISBOOL(type_num1) ||
+        if (type_num1 < NPY_NTYPES_LEGACY && !(PyTypeNum_ISBOOL(type_num1) ||
                                         PyTypeNum_ISUNSIGNED(type_num1))) {
             /* Convert to the equivalent-sized signed integer */
             type_num2 = type_num_unsigned_to_signed(type_num2);
@@ -1881,7 +1881,7 @@ PyArray_ResultType(
      * 2. It does nothing, but warns if there the result would differ.
      * 3. It replaces the result based on the legacy value-based logic.
      */
-    if (at_least_one_scalar && !all_pyscalar && result->type_num < NPY_NTYPES) {
+    if (at_least_one_scalar && !all_pyscalar && result->type_num < NPY_NTYPES_LEGACY) {
         if (PyArray_CheckLegacyResultType(
                 &result, narrs, arrs, ndtypes, descrs) < 0) {
             Py_DECREF(common_dtype);
@@ -2701,13 +2701,13 @@ add_numeric_cast(PyArray_DTypeMeta *from, PyArray_DTypeMeta *to)
 static int
 PyArray_InitializeNumericCasts(void)
 {
-    for (int from = 0; from < NPY_NTYPES; from++) {
+    for (int from = 0; from < NPY_NTYPES_LEGACY; from++) {
         if (!PyTypeNum_ISNUMBER(from) && from != NPY_BOOL) {
             continue;
         }
         PyArray_DTypeMeta *from_dt = PyArray_DTypeFromTypeNum(from);
 
-        for (int to = 0; to < NPY_NTYPES; to++) {
+        for (int to = 0; to < NPY_NTYPES_LEGACY; to++) {
             if (!PyTypeNum_ISNUMBER(to) && to != NPY_BOOL) {
                 continue;
             }
@@ -2971,7 +2971,7 @@ PyArray_InitializeStringCasts(void)
     PyArray_DTypeMeta *other_dt = NULL;
 
     /* Add most casts as legacy ones */
-    for (int other = 0; other < NPY_NTYPES; other++) {
+    for (int other = 0; other < NPY_NTYPES_LEGACY; other++) {
         if (PyTypeNum_ISDATETIME(other) || other == NPY_VOID ||
                 other == NPY_OBJECT) {
             continue;

numpy/_core/src/multiarray/datetime.c

@@ -4156,7 +4156,7 @@ PyArray_InitializeDatetimeCasts()
      * Some of these casts can fail (casting to unitless datetime), but these
      * are rather special.
      */
-    for (int num = 0; num < NPY_NTYPES; num++) {
+    for (int num = 0; num < NPY_NTYPES_LEGACY; num++) {
         if (!PyTypeNum_ISNUMBER(num) && num != NPY_BOOL) {
             continue;
         }

numpy/_core/src/multiarray/dtypemeta.c

@@ -918,7 +918,7 @@ string_known_scalar_types(
 static PyArray_DTypeMeta *
 default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 {
-    assert(cls->type_num < NPY_NTYPES);
+    assert(cls->type_num < NPY_NTYPES_LEGACY);
     if (NPY_UNLIKELY(NPY_DT_is_abstract(other))) {
         /*
          * The abstract complex has a lower priority than the concrete inexact
@@ -982,7 +982,7 @@ default_builtin_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 static PyArray_DTypeMeta *
 string_unicode_common_dtype(PyArray_DTypeMeta *cls, PyArray_DTypeMeta *other)
 {
-    assert(cls->type_num < NPY_NTYPES && cls != other);
+    assert(cls->type_num < NPY_NTYPES_LEGACY && cls != other);
     if (!NPY_DT_is_legacy(other) || (!PyTypeNum_ISNUMBER(other->type_num) &&
             /* Not numeric so defer unless cls is unicode and other is string */
             !(cls->type_num == NPY_UNICODE && other->type_num == NPY_STRING))) {
@@ -1078,7 +1078,7 @@ dtypemeta_wrap_legacy_descriptor(PyArray_Descr *descr,
 
     if (!has_type_set) {
         /* Accept if the type was filled in from an existing builtin dtype */
-        for (int i = 0; i < NPY_NTYPES; i++) {
+        for (int i = 0; i < NPY_NTYPES_LEGACY; i++) {
             PyArray_Descr *builtin = PyArray_DescrFromType(i);
             has_type_set = Py_TYPE(descr) == Py_TYPE(builtin);
             Py_DECREF(builtin);