ENH: meson: implement BLAS/LAPACK auto-detection and many CI jobs [wh…

…eel build]

This reimplements the auto-detection and many of the switches that
`numpy.distutils` offered. Beyond that, it implements several new
features:

- Auto-detect the symbol suffix for ILP64 OpenBLAS (can be none or
  `64_`)
- MKL ILP64 support, threading control, and use of the layered library
  model for MKL >=2023.0
- FlexiBLAS support (LP64 and ILP64)
- Support for the upcoming standard in Reference LAPACK for `_64` ILP64
  symbol suffix convention.
- A test suite for BLAS/LAPACK libraries, covering:

    - OpenBLAS: LP64, ILP64 detected via pkg-config and with a "system
                dependency" (i.e., custom code inside Meson)
    - MKL: LP64, ILP64 (layered) and LP64 (SDL)
    - Accelerate: LP64, ILP64 on macOS >=13.3
    - FlexiBLAS: LP64, ILP64 on Fedora
    - ATLAS (LP64, via pkg-config only)
    - BLIS (LP64, via pkg-config only)
    - plain libblas/liblapack (Netlib, LP64 only)

The list of libraries that is tried with the default 'auto' setting
excludes a couple of libraries, because they're either no longer
developed (ATLAS), not mature (libflame), or can't be tested and
may be re-added later (ArmPL, ssl2). Those libraries can still be
quite easily used via pkg-config.

The new CI jobs are running by default right now. Once things settle
down, the plan is to disable them by default and allow triggering
them via a `[blas ci]` command in the commit message (just like for
wheel builds).

Docs will be included in a separate PR with the pending rewrite of
all the build/install docs. For now, the CI jobs and the
`meson_options.txt` file serve as guidance for how to use this.

Note that the test suite contains a few hacks, because of packaging
bugs for MKL on PyPI (broken .pc files) and BLIS (missing .pc file in
Debian).

.github/workflows/linux_blas.yml

@@ -10,9 +10,27 @@ name: BLAS tests (Linux)
 #   - openblas32_stable_nightly:
 #         Uses the 32-bit OpenBLAS builds, both the latest stable release
 #         and a nightly build.
-#
-# TODO: coverage here is limited, we should add non-OpenBLAS libraries and
-#       exercise the BLAS-related build options (see `meson_options.txt`).
+#   - openblas_no_pkgconfig_fedora:
+#         Test OpenBLAS on Fedora. Fedora doesn't ship .pc files for OpenBLAS,
+#         hence this exercises the "system dependency" detection method.
+#   - flexiblas_fedora:
+#         Tests FlexiBLAS (the default on Fedora for its own packages), via
+#         pkg-config. FlexiBLAS allows runtime switching of BLAS/LAPACK
+#         libraries, which is a useful capability (not tested in this job).
+#   - openblas_cmake:
+#         Tests whether OpenBLAS LP64 is detected correctly when only CMake
+#         and not pkg-config is installed.
+#   - netlib:
+#         Installs vanilla blas/lapack, which is the last option tried in
+#         auto-detection.
+#   - mkl:
+#         Tests MKL installed from PyPI (because easiest/fastest, if broken) in
+#         3 ways: both LP64 and ILP64 via pkg-config, and then using the
+#         Single Dynamic Library (SDL, or `libmkl_rt`).
+#   - blis:
+#         Simple test for LP64 via pkg-config
+#   - atlas:
+#         Simple test for LP64 via pkg-config
 
 on:
   pull_request:
@@ -87,3 +105,253 @@ jobs:
       run: |
         pip install pytest pytest-xdist hypothesis typing_extensions
         spin test -j auto
+
+
+  openblas_no_pkgconfig_fedora:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    container: fedora:39
+    name: "OpenBLAS (Fedora, no pkg-config, LP64/ILP64)"
+    steps:
+    - name: Install system dependencies
+      run: |
+        dnf install git gcc-gfortran g++ python3-devel openblas-devel -y
+
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest hypothesis typing_extensions
+
+    - name: Build (LP64)
+      run: spin build -- -Dblas=openblas -Dlapack=openblas -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+    - name: Build (ILP64)
+      run: |
+        rm -rf build
+        spin build -- -Duse-ilp64=true -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+
+  flexiblas_fedora:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    container: fedora:39
+    name: "FlexiBLAS (LP64, ILP64 on Fedora)"
+    steps:
+    - name: Install system dependencies
+      run: |
+        dnf install git gcc-gfortran g++ python3-devel flexiblas-devel -y
+
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest hypothesis typing_extensions
+
+    - name: Build
+      run: spin build -- -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+    - name: Build (ILP64)
+      run: |
+        rm -rf build
+        spin build -- -Ddisable-optimization=true -Duse-ilp64=true
+
+    - name: Test (ILP64)
+      run: spin test -- numpy/linalg
+
+
+  openblas_cmake:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    name: "OpenBLAS with CMake"
+    steps:
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest pytest-xdist hypothesis typing_extensions
+        sudo apt-get install libopenblas-dev cmake
+        sudo apt-get remove pkg-config
+
+    - name: Build
+      run: spin build -- -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -j auto -- numpy/linalg
+
+
+  netlib:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    name: "Netlib BLAS/LAPACK"
+    steps:
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        sudo apt-get install liblapack-dev pkg-config
+
+    - name: Build
+      run: |
+        spin build -- -Ddisable-optimization=true
+
+    - name: Test
+      run: |
+        pip install pytest pytest-xdist hypothesis typing_extensions
+        spin test -j auto -- numpy/linalg
+
+
+  mkl:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    name: "MKL (LP64, ILP64, SDL)"
+    steps:
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest pytest-xdist hypothesis typing_extensions
+        pip install mkl mkl-devel
+
+    - name: Repair MKL pkg-config files and symlinks
+      run: |
+        # MKL 2023.2 works when installed from conda-forge (except for `-iomp`
+        # and `-tbb` pkg-config files), Spack, or with the standalone Intel
+        # installer. The standalone installer is the worst option, since it's
+        # large and clumsy to install and requires running a setvars.sh script
+        # before things work. The PyPI MKL packages are broken and need the
+        # fixes in this step. For details, see
+        # https://github.com/conda-forge/intel_repack-feedstock/issues/34
+        cd $Python3_ROOT_DIR/lib/pkgconfig
+        sed -i 's/\/intel64//g' mkl*.pc
+        # add the expected .so -> .so.2 symlinks to fix linking
+        cd ..
+        for i in $( ls libmkl*.so.2 ); do ln -s $i ${i%.*}; done
+
+    - name: Build with defaults (LP64)
+      run: |
+        pkg-config --libs mkl-dynamic-lp64-seq  # check link flags
+        spin build -- -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+    - name: Build with ILP64
+      run: |
+        git clean -xdf > /dev/null
+        pkg-config --libs mkl-dynamic-ilp64-seq
+        spin build -- -Duse-ilp64=true -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+    - name: Build without pkg-config (default options, SDL)
+      run: |
+        git clean -xdf > /dev/null
+        pushd $Python3_ROOT_DIR/lib/pkgconfig
+        rm mkl*.pc
+        popd
+        export MKLROOT=$Python3_ROOT_DIR
+        spin build -- -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+  blis:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    name: "BLIS"
+    steps:
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest pytest-xdist hypothesis typing_extensions
+        sudo apt-get install libblis-dev libopenblas-dev pkg-config
+
+    - name: Add BLIS pkg-config file
+      run: |
+        # Needed because blis.pc missing in Debian:
+        # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=989076
+        # The alternative here would be to use another distro or Miniforge
+        sudo cp tools/ci/_blis_debian.pc /usr/lib/x86_64-linux-gnu/pkgconfig/blis.pc
+        # Check if the patch works:
+        pkg-config --libs blis
+        pkg-config --cflags blis
+
+    - name: Build
+      run: spin build -- -Dblas=blis -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+
+  atlas:
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: ubuntu-latest
+    name: "ATLAS"
+    steps:
+    - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.11'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest pytest-xdist hypothesis typing_extensions
+        sudo apt-get install libatlas-base-dev pkg-config
+
+    - name: Build
+      run: spin build -- -Dblas=blas-atlas -Dlapack=lapack-atlas -Ddisable-optimization=true
+
+    - name: Test
+      run: spin test -- numpy/linalg
+

.github/workflows/linux_musl.yml

@@ -60,7 +60,7 @@ jobs:
         pip install -r build_requirements.txt -r test_requirements.txt
 
         # use meson to build and test 
-        spin build --with-scipy-openblas=64
+        spin build --with-scipy-openblas=64 -- -Duse-ilp64=true
         spin test -j auto
 
     - name: Meson Log

.github/workflows/macos.yml

@@ -100,7 +100,7 @@ jobs:
         ccache -s
 
   accelerate:
-    name: Accelerate ILP64
+    name: Accelerate (LP64, ILP64)
     if: "github.repository == 'numpy/numpy'"
     runs-on: macos-13
     steps:
@@ -122,14 +122,15 @@ jobs:
         pip install -r build_requirements.txt
         pip install pytest pytest-xdist hypothesis
 
-    - name: Build NumPy against Accelerate (ILP64)
-      run: |
-        spin build -- -Dblas=accelerate -Dlapack=accelerate -Duse-ilp64=true
+    - name: Build against Accelerate (LP64)
+      run: spin build -- -Ddisable-optimization=true
 
-    - name: Show meson-log.txt
-      if: always()
-      run: 'cat build/meson-logs/meson-log.txt'
+    - name: Test (linalg only)
+      run: spin test -j2 -- numpy/linalg
 
-    - name: Test
+    - name: Build NumPy against Accelerate (ILP64)
       run: |
-        spin test -j2
+        spin build -- -Duse-ilp64=true
+
+    - name: Test (fast tests)
+      run: spin test -j2

azure-steps-windows.yml

@@ -31,7 +31,7 @@ steps:
     }
     elseif ( Test-Path env:NPY_USE_BLAS_ILP64 ) {
         python -m pip install scipy-openblas64 spin
-        spin config-openblas --with-scipy-openblas=64
+        spin config-openblas --with-scipy-openblas=64 -- -Duse-ilp64=true
         $env:PKG_CONFIG_PATH="$pwd/.openblas"
         python -m pip install . -v -Csetup-args="--vsenv"
     } else {

meson.build

@@ -12,8 +12,6 @@ project(
     'b_ndebug=if-release',
     'c_std=c99',
     'cpp_std=c++17',
-    'blas=openblas',
-    'lapack=openblas',
     'pkgconfig.relocatable=true',
   ],
 )

meson_options.txt

@@ -1,13 +1,19 @@
-option('blas', type: 'string', value: 'openblas',
-        description: 'Option for BLAS library switching')
-option('lapack', type: 'string', value: 'openblas',
-        description: 'Option for LAPACK library switching')
+option('blas', type: 'string', value: 'auto',
+        description: 'Option for BLAS library selection. By default, try to find any in the order given by `blas-order`')
+option('lapack', type: 'string', value: 'auto',
+        description: 'Option for LAPACK library selection. By default, try to find any in the order given by `lapack-order`')
 option('allow-noblas', type: 'boolean', value: false,
         description: 'If set to true, allow building with (slow!) internal fallback routines')
+option('blas-order', type: 'array',
+        value: ['mkl', 'accelerate', 'openblas', 'flexiblas', 'blis', 'blas'])
+option('lapack-order', type: 'array',
+        value: ['mkl', 'accelerate', 'openblas', 'flexiblas', 'lapack'])
 option('use-ilp64', type: 'boolean', value: false,
        description: 'Use ILP64 (64-bit integer) BLAS and LAPACK interfaces')
-option('blas-symbol-suffix', type: 'string', value: '',
-        description: 'BLAS and LAPACK symbol suffix to use, if any (often `64_` for ILP64)')
+option('blas-symbol-suffix', type: 'string', value: 'auto',
+        description: 'BLAS and LAPACK symbol suffix to use, if any')
+option('mkl-threading', type: 'string', value: 'auto',
+        description: 'MKL threading method, one of: `seq`, `iomp`, `gomp`, `tbb`')
 option('disable-svml', type: 'boolean', value: false,
         description: 'Disable building against SVML')
 option('disable-threading', type: 'boolean', value: false,

numpy/core/src/common/npy_cblas.h

@@ -57,8 +57,21 @@ enum CBLAS_SIDE {CblasLeft=141, CblasRight=142};
 #define BLAS_FUNC_CONCAT(name,prefix,suffix,suffix2) prefix ## name ## suffix ## suffix2
 #define BLAS_FUNC_EXPAND(name,prefix,suffix,suffix2) BLAS_FUNC_CONCAT(name,prefix,suffix,suffix2)
 
-#define CBLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,,BLAS_SYMBOL_SUFFIX)
+/*
+ * Use either the OpenBLAS scheme with the `64_` suffix behind the Fortran
+ * compiler symbol mangling, or the MKL scheme (and upcoming
+ * reference-lapack#666) which does it the other way around and uses `_64`.
+ */
+#ifdef OPENBLAS_ILP64_NAMING_SCHEME
 #define BLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,BLAS_FORTRAN_SUFFIX,BLAS_SYMBOL_SUFFIX)
+#else
+#define BLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,BLAS_SYMBOL_SUFFIX,BLAS_FORTRAN_SUFFIX)
+#endif
+/*
+ * Note that CBLAS doesn't include Fortran compiler symbol mangling, so ends up
+ * being the same in both schemes
+ */
+#define CBLAS_FUNC(name) BLAS_FUNC_EXPAND(name,BLAS_SYMBOL_PREFIX,,BLAS_SYMBOL_SUFFIX)
 
 #ifdef HAVE_BLAS_ILP64
 #define CBLAS_INT npy_int64