BLD: use scipy-openblas wheel (numpy#24839)

.github/meson_actions/action.yml

@@ -7,6 +7,7 @@ runs:
     shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
     env:
       TERM: xterm-256color
+      PKG_CONFIG_PATH: ./.openblas
     run: |
       echo "::group::Installing Build Dependencies"
       pip install -r build_requirements.txt

.github/workflows/linux.yml

@@ -73,10 +73,10 @@ jobs:
     - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       with:
         python-version: 'pypy3.9-v7.3.12'
-    - name: Install system dependencies
+    - name: Setup using scipy-openblas
       run: |
-        sudo apt-get update
-        sudo apt-get install libopenblas-dev ninja-build
+        python -m pip install scipy-openblas32 spin
+        spin config-openblas --with-scipy-openblas=32
     - uses: ./.github/meson_actions
 
   debug:
@@ -124,15 +124,16 @@ jobs:
       run: |
         pip install -r build_requirements.txt
         pip install -r test_requirements.txt
-    - name: Install gfortran and OpenBLAS (MacPython build)
+    - name: Install gfortran and setup OpenBLAS (MacPython build)
       run: |
         set -xe
         sudo apt update
         sudo apt install gfortran libgfortran5
-        target=$(python tools/openblas_support.py)
-        sudo cp -r $target/lib/* /usr/lib
-        sudo cp $target/include/* /usr/include
+        pip install scipy-openblas32
+        spin config-openblas --with-scipy-openblas=32
     - name: Build a wheel
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
         python -m build --wheel --no-isolation --skip-dependency-check
         pip install dist/numpy*.whl
@@ -185,7 +186,7 @@ jobs:
     - uses: actions/setup-python@65d7f2d534ac1bc67fcd62888c5f4f3d2cb2b236 # v4.7.1
       with:
         python-version: '3.11'
-    - name: Install gfortran and OpenBLAS (MacPython build)
+    - name: Install gfortran and setup OpenBLAS (MacPython build)
       run: |
         set -xe
         sudo apt update

.github/workflows/linux_blas.yml

@@ -7,13 +7,9 @@ name: BLAS tests (Linux)
 #
 # Jobs and their purpose:
 #
-#   - openblas64_setuppy:
-#         This job uses the default 64-bit build of OpenBLAS with the
-#         `numpy.distutils`-based build. It can be removed once we remove
-#         support for those builds.
 #   - openblas32_stable_nightly:
-#         Uses the 32-bit OpenBLAS builds, both the latest stable release and a
-#         nightly build.
+#         Uses the 32-bit OpenBLAS builds, both the latest stable release
+#         and a nightly build.
 #
 # TODO: coverage here is limited, we should add non-OpenBLAS libraries and
 #       exercise the BLAS-related build options (see `meson_options.txt`).

.github/workflows/linux_compiler_sanitizers.yml

@@ -35,14 +35,14 @@ jobs:
     - name: Install dependencies
       run: |
         pip install -r build_requirements.txt
-        sudo apt-get update
-        sudo apt-get install -y libopenblas-serial-dev
+        pip install scipy-openblas32 spin
     - name: Build
       shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
       env:
         TERM: xterm-256color
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run:
-        spin build -- --werror -Db_sanitize=address,undefined
+        spin build --with-scipy-openblas=32 -- --werror -Db_sanitize=address,undefined
     - name: Test
       shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"'
       env:

.github/workflows/linux_musl.yml

@@ -48,20 +48,22 @@ jobs:
 
         ln -s /usr/local/bin/python3.10 /usr/local/bin/python
 
-    - name: test musllinux_x86_64
+    - name: test-musllinux_x86_64
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
         python -m venv test_env
         source test_env/bin/activate
 
-        # required for figuring out the system tags in openblas_support
-        pip install packaging       
-        
-        # install openblas by co-opting the CIBW setup script
-        RUNNER_OS=Linux sh tools/wheels/cibw_before_build.sh .
+        pip install scipy-openblas64
 
-        pip install -r build_requirements.txt
-        pip install pytest pytest-xdist hypothesis typing_extensions
+        pip install -r build_requirements.txt -r test_requirements.txt
 
         # use meson to build and test 
-        spin build
+        spin build --with-scipy-openblas=64
         spin test -j auto
+
+    - name: Meson Log
+      shell: bash
+      run: |
+        cat build/meson-logs/meson-log.txt

.github/workflows/windows.yml

@@ -37,16 +37,9 @@ jobs:
       run: |
         python -m pip install spin Cython
 
-    - name: Install OpenBLAS (MacPython build)
+    - name: Install pkg-config
       run: |
-        # Download and install pre-built OpenBLAS library with 32-bit
-        # interfaces. Unpack it in the pkg-config hardcoded path
-        choco install unzip -y
-        choco install wget -y
         choco install -y --checksum 6004DF17818F5A6DBF19CB335CC92702 pkgconfiglite
-        wget https://anaconda.org/multibuild-wheels-staging/openblas-libs/v0.3.21/download/openblas-v0.3.21-win_amd64-gcc_10_3_0.zip
-        unzip -d c:\opt openblas-v0.3.21-win_amd64-gcc_10_3_0.zip
-        echo "PKG_CONFIG_PATH=c:\opt\64\lib\pkgconfig;" >> $env:GITHUB_ENV
 
     - name: Install Clang-cl
       if: matrix.compiler == 'Clang-cl'
@@ -55,26 +48,26 @@ jobs:
 
     - name: Install NumPy (MSVC)
       if: matrix.compiler == 'MSVC'
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
-        spin build -j2 -- --vsenv
+        python -m pip install scipy-openblas32
+        spin build --with-scipy-openblas=32 -j2 -- --vsenv
 
     - name: Install NumPy (Clang-cl)
       if: matrix.compiler == 'Clang-cl'
+      env:
+        PKG_CONFIG_PATH: ${{ github.workspace }}/.openblas
       run: |
         "[binaries]","c = 'clang-cl'","cpp = 'clang-cl'","ar = 'llvm-lib'","c_ld = 'lld-link'","cpp_ld = 'lld-link'" | Out-File $PWD/clang-cl-build.ini -Encoding ascii
-        spin build -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini
+        python -m pip install scipy-openblas32
+        spin build --with-scipy-openblas=32 -j2 -- --vsenv --native-file=$PWD/clang-cl-build.ini
 
-    - name: Copy OpenBLAS DLL, write _distributor_init.py
+    - name: Meson Log
+      shell: bash
+      if: ${{ failure()  }}
       run: |
-        # Getting the OpenBLAS DLL to the right place so it loads
-        $installed_path = "$PWD\build-install\usr\Lib\site-packages"
-        $numpy_path = "${installed_path}\numpy"
-        $libs_path = "${installed_path}\numpy.libs"
-        mkdir ${libs_path}
-        $ob_path = "C:/opt/64/bin/"
-        cp $ob_path/*.dll $libs_path
-        # Write _distributor_init.py to load .libs DLLs.
-        python -c "from tools import openblas_support; openblas_support.make_init(r'${numpy_path}')"
+        cat build/meson-logs/meson-log.txt
 
     - name: Install test dependencies
       run: |
@@ -85,7 +78,7 @@ jobs:
       run: |
         spin test
 
-  msvc_32bit_python_openblas:
+  msvc_32bit_python_no_openblas:
     name: MSVC, 32-bit Python, no BLAS
     runs-on: windows-2019
     if: "github.repository == 'numpy/numpy'"

.gitignore

@@ -118,7 +118,6 @@ doc/source/**/generated/
 
 # Things specific to this project #
 ###################################
-# The line below should change to numpy/_version.py for NumPy 2.0
 benchmarks/results
 benchmarks/html
 benchmarks/env
@@ -139,3 +138,5 @@ tools/swig/test/Tensor.py
 tools/swig/test/Vector.py
 tools/swig/test/Vector_wrap.cxx
 tools/swig/test/Array.py
+.openblas
+numpy/_distributor_init_local.py

.spin/cmds.py

@@ -37,9 +37,14 @@
     "-v", "--verbose", is_flag=True,
     help="Print all build output, even installation"
 )
+@click.option(
+    "--with-scipy-openblas", type=click.Choice(["32", "64"]),
+    default=None,
+    help="Build with pre-installed scipy-openblas32 or scipy-openblas64 wheel"
+)
 @click.argument("meson_args", nargs=-1)
 @click.pass_context
-def build(ctx, meson_args, jobs=None, clean=False, verbose=False, quiet=False):
+def build(ctx, meson_args, with_scipy_openblas, jobs=None, clean=False, verbose=False, quiet=False):
     """🔧 Build package with Meson/ninja and install
 
     MESON_ARGS are passed through e.g.:
@@ -53,6 +58,10 @@ def build(ctx, meson_args, jobs=None, clean=False, verbose=False, quiet=False):
 
     CFLAGS="-O0 -g" spin build
     """
+    # XXX keep in sync with upstream build
+    if with_scipy_openblas:
+        _config_openblas(with_scipy_openblas)
+    ctx.params.pop("with_scipy_openblas", None)
     ctx.forward(meson.build)
 
 
@@ -413,7 +422,7 @@ def ipython(ctx, ipython_args):
     util.run(["ipython", "--ignore-cwd",
               f"--TerminalIPythonApp.exec_lines={preimport}"] +
              list(ipython_args))
- 
+
 
 @click.command(context_settings={"ignore_unknown_options": True})
 @click.pass_context
@@ -425,3 +434,39 @@ def mypy(ctx):
     ctx.params['pytest_args'] = [os.path.join('numpy', 'typing')]
     ctx.params['markexpr'] = 'full'
     ctx.forward(test)
+
+@click.command(context_settings={
+    'ignore_unknown_options': True
+})
+@click.option(
+    "--with-scipy-openblas", type=click.Choice(["32", "64"]),
+    default=None, required=True,
+    help="Build with pre-installed scipy-openblas32 or scipy-openblas64 wheel"
+)
+def config_openblas(with_scipy_openblas):
+    """🔧 Create .openblas/scipy-openblas.pc file
+
+    Also create _distributor_init_local.py
+
+    Requires a pre-installed scipy-openblas64 or scipy-openblas32
+    """
+    _config_openblas(with_scipy_openblas)
+
+
+def _config_openblas(blas_variant):
+    import importlib
+    basedir = os.getcwd()
+    openblas_dir = os.path.join(basedir, ".openblas")
+    pkg_config_fname = os.path.join(openblas_dir, "scipy-openblas.pc")
+    if blas_variant:
+        module_name = f"scipy_openblas{blas_variant}"
+        try:
+            openblas = importlib.import_module(module_name)
+        except ModuleNotFoundError:
+            raise RuntimeError(f"'pip install {module_name} first")
+        local = os.path.join(basedir, "numpy", "_distributor_init_local.py")
+        with open(local, "wt", encoding="utf8") as fid:
+            fid.write(f"import {module_name}\n")
+        os.makedirs(openblas_dir, exist_ok=True)
+        with open(pkg_config_fname, "wt", encoding="utf8") as fid:
+            fid.write(openblas.get_pkg_config().replace("\\", "/"))

azure-pipelines.yml

@@ -73,13 +73,12 @@ stages:
             cd /numpy && \
             /opt/python/cp39-cp39/bin/python -mvenv venv && \
             source venv/bin/activate && \
-            target=\$(python3 tools/openblas_support.py) && \
-            cp -r \$target/lib/* /usr/lib && \
-            cp \$target/include/* /usr/include && \
-            python3 -m pip install ninja && \
+            python3 -m pip install ninja scipy-openblas32 spin && \
             python3 -m pip install -r test_requirements.txt && \
             echo CFLAGS \$CFLAGS && \
-            python3 -m pip install -v . && \
+            spin config-openblas --with-scipy-openblas=32 && \
+            export PKG_CONFIG_PATH=/numpy/.openblas && \
+            python3 -m pip install  . && \
             cd tools && \
             python3 -m pytest --pyargs numpy"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'
@@ -107,8 +106,6 @@ stages:
             TEST_MODE: fast
             BITS: 64
             NPY_USE_BLAS_ILP64: '1'
-            # Broken - it builds but _multiarray_umath doesn't import - needs investigating
-            DISABLE_BLAS: '1'
 
     steps:
     - template: azure-steps-windows.yml

azure-steps-windows.yml

@@ -24,43 +24,24 @@ steps:
   displayName: 'Install utilities'
 
 - powershell: |
-    $ErrorActionPreference = "Stop"
-    mkdir  C:/opt/openblas/openblas_dll
-    mkdir  C:/opt/32/lib/pkgconfig
-    mkdir  C:/opt/64/lib/pkgconfig
-    $target=$(python -c "import tools.openblas_support as obs; plat=obs.get_plat(); ilp64=obs.get_ilp64(); target=f'openblas_{plat}.zip'; obs.download_openblas(target, plat, ilp64);print(target)")
-    unzip -o -d c:/opt/ $target
-    echo "##vso[task.setvariable variable=PKG_CONFIG_PATH]c:/opt/64/lib/pkgconfig"
-    copy C:/opt/64/bin/*.dll C:/opt/openblas/openblas_dll
-  displayName: 'Download / Install OpenBLAS'
-
-- powershell: |
-    # Note: ensure the `pip install .` command remains the last one here, to
-    # avoid "green on failure" issues
-    python -c "from tools import openblas_support; openblas_support.make_init('numpy')"
+    # Note: ensure the `pip install .` command remains the last one here,
+    # to avoid "green on failure" issues
     If ( Test-Path env:DISABLE_BLAS ) {
         python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Dblas=none" -Csetup-args="-Dlapack=none" -Csetup-args="-Dallow-noblas=true"
     }
     elseif ( Test-Path env:NPY_USE_BLAS_ILP64 ) {
-        python -m pip install . -v -Csetup-args="--vsenv" -Csetup-args="-Duse-ilp64=true" -Csetup-args="-Dblas-symbol-suffix=64_"
-    } else {
+        python -m pip install scipy-openblas64 spin
+        spin config-openblas --with-scipy-openblas=64
+        $env:PKG_CONFIG_PATH="$pwd/.openblas"
         python -m pip install . -v -Csetup-args="--vsenv"
+    } else {
+        python -m pip install scipy-openblas32 spin
+        spin config-openblas --with-scipy-openblas=32
+        $env:PKG_CONFIG_PATH="$pwd/.openblas"
+        python -m pip install . -v -Csetup-args="--vsenv" 
     }
   displayName: 'Build NumPy'
 
-- powershell: |
-    # copy from c:/opt/openblas/openblas_dll to numpy/../numpy.libs to ensure it can
-    # get loaded when numpy is imported (no RPATH on Windows)
-    $target = $(python -c "import sysconfig; print(sysconfig.get_path('platlib'))")
-    mkdir $target/numpy.libs
-    copy C:/opt/openblas/openblas_dll/*.dll $target/numpy.libs
-  displayName: 'Copy OpenBLAS DLL to site-packages'
-
-- script: |
-    python -m pip install threadpoolctl
-    python tools/openblas_support.py --check_version
-  displayName: 'Check OpenBLAS version'
-
 - powershell: |
     cd tools  # avoid root dir to not pick up source tree
     # Get a gfortran onto the path for f2py tests

doc/source/dev/development_environment.rst

@@ -66,6 +66,10 @@ Now, whenever you want to switch to the virtual environment, you can use the
 command ``source numpy-dev/bin/activate``, and ``deactivate`` to exit from the
 virtual environment and back to your previous shell.
 
+Building from source
+--------------------
+
+See building-from-source_
 
 .. _testing-builds:
 

numpy/_distributor_init.py

@@ -3,8 +3,13 @@
 Distributors: you can add custom code here to support particular distributions
 of numpy.
 
-For example, this is a good place to put any checks for hardware requirements.
+For example, this is a good place to put any BLAS/LAPACK initialization code.
 
 The numpy standard source distribution will not put code in this file, so you
 can safely replace this file with your own version.
 """
+
+try:
+    from . import _distributor_init_local
+except ImportError:
+    pass