Merge pull request numpy#13920 from mattip/openblas

ENH, BUILD: refactor all OpenBLAS downloads into a single, testable file

azure-pipelines.yml

@@ -20,21 +20,17 @@ jobs:
            docker pull i386/ubuntu:bionic
            docker run -v $(pwd):/numpy i386/ubuntu:bionic /bin/bash -c "cd numpy && \
            apt-get -y update && \
-           apt-get -y install python3.6-dev python3-pip locales && \
+           apt-get -y install python3.6-dev python3-pip locales python3-certifi && \
            locale-gen fr_FR && update-locale && \
            pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \
            apt-get -y install gfortran-5 wget && \
-           cd .. && \
-           mkdir openblas && cd openblas && \
-           wget https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \
-           tar zxvf openblas-v0.3.5-274-g6a8b4269-manylinux1_i686.tar.gz && \
-           cp -r ./usr/local/lib/* /usr/lib && \
-           cp ./usr/local/include/* /usr/include && \
-           cd ../numpy && \
+           target=\$(python3 tools/openblas_support.py) && \
+           cp -r \$target/usr/local/lib/* /usr/lib && \
+           cp \$target/usr/local/include/* /usr/include && \
            python3 -m pip install . && \
            F77=gfortran-5 F90=gfortran-5 \
            CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \
-           cd ../openblas && python3 -c \"$(TEST_GET_CONFIG)\""
+           cd .. && python3 -c \"$(TEST_GET_CONFIG)\""
     displayName: 'Run 32-bit Ubuntu Docker Build / Tests'
   - task: PublishTestResults@2
     condition: succeededOrFailed()
@@ -83,11 +79,10 @@ jobs:
   # matches our MacOS wheel builds -- currently based
   # primarily on file size / name details
   - script: |
-      wget "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-macosx_10_9_x86_64-gf_1becaaa.tar.gz"
-      tar -zxvf openblas-v0.3.5-274-g6a8b4269-macosx_10_9_x86_64-gf_1becaaa.tar.gz
+      target=$(python tools/openblas_support.py)
       # manually link to appropriate system paths
-      cp ./usr/local/lib/* /usr/local/lib/
-      cp ./usr/local/include/* /usr/local/include/
+      cp $target/usr/local/lib/* /usr/local/lib/
+      cp $target/usr/local/include/* /usr/local/include/
     displayName: 'install pre-built openblas'
   - script: python -m pip install --upgrade pip setuptools wheel
     displayName: 'Install tools'
@@ -124,45 +119,35 @@ jobs:
 - job: Windows
   pool:
     vmImage: 'VS2017-Win2016'
-  variables:
-      # openblas URLs from numpy-wheels
-      # appveyor / Windows config
-      OPENBLAS_32: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win32-gcc_7_1_0.zip"
-      OPENBLAS_64: "https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/openblas-v0.3.5-274-g6a8b4269-win_amd64-gcc_7_1_0.zip"
   strategy:
     maxParallel: 6
     matrix:
         Python36-32bit-fast:
           PYTHON_VERSION: '3.6'
           PYTHON_ARCH: 'x86'
           TEST_MODE: fast
-          OPENBLAS: $(OPENBLAS_32)
           BITS: 32
         Python37-32bit-fast:
           PYTHON_VERSION: '3.7'
           PYTHON_ARCH: 'x86'
           TEST_MODE: fast
-          OPENBLAS: $(OPENBLAS_32)
           BITS: 32
         Python35-64bit-full:
           PYTHON_VERSION: '3.5'
           PYTHON_ARCH: 'x64'
           TEST_MODE: full
-          OPENBLAS: $(OPENBLAS_64)
           BITS: 64
         Python36-64bit-full:
           PYTHON_VERSION: '3.6'
           PYTHON_ARCH: 'x64'
           TEST_MODE: full
           INSTALL_PICKLE5: 1
-          OPENBLAS: $(OPENBLAS_64)
           BITS: 64
         Python37-64bit-full:
           PYTHON_VERSION: '3.7'
           PYTHON_ARCH: 'x64'
           TEST_MODE: full
           INSTALL_PICKLE5: 1
-          OPENBLAS: $(OPENBLAS_64)
           BITS: 64
   steps:
   - task: UsePythonVersion@0
@@ -176,17 +161,16 @@ jobs:
     displayName: 'Install dependencies; some are optional to avoid test skips'
   - script: if [%INSTALL_PICKLE5%]==[1] python -m pip install pickle5
     displayName: 'Install optional pickle5 backport (only for python3.6 and 3.7)'
+
   - powershell: |
-      $wc = New-Object net.webclient
-      $wc.Downloadfile("$(OPENBLAS)", "openblas.zip")
-      $tmpdir = New-TemporaryFile | %{ rm $_; mkdir $_ }
-      Expand-Archive "openblas.zip" $tmpdir
       $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])"
       Write-Host "Python Version: $pyversion"
       $target = "C:\\hostedtoolcache\\windows\\Python\\$pyversion\\$(PYTHON_ARCH)\\lib\\openblas.a"
       Write-Host "target path: $target"
-      cp $tmpdir\$(BITS)\lib\libopenblas_v0.3.5-274-g6a8b4269-gcc_7_1_0.a $target
+      $openblas = python tools/openblas_support.py
+      cp $openblas $target
     displayName: 'Download / Install OpenBLAS'
+
   - powershell: |
       choco install -y mingw --forcex86 --force --version=5.3.0
     displayName: 'Install 32-bit mingw for 32-bit builds'

shippable.yml

@@ -23,15 +23,11 @@ build:
     ci:
     # install dependencies
     - sudo apt-get install gcc gfortran
-    # ARMv8 OpenBLAS built using script available here:
-    # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8
-    # build done on GCC compile farm machine named gcc115
-    # tarball uploaded manually to an unshared Dropbox location
-    - wget -O openblas-v0.3.5-armv8.tar.gz https://www.dropbox.com/s/pbqkxzlmih4cky1/openblas-v0.3.5-armv8.tar.gz?dl=0
-    - tar zxvf openblas-v0.3.5-armv8.tar.gz
-    - sudo cp -r ./64/lib/* /usr/lib
-    - sudo cp ./64/include/* /usr/include
+    - target=$(python tools/openblas_support.py)
+    - sudo cp -r "${target}"/64/lib/* /usr/lib
+    - sudo cp "${target}"/64/include/* /usr/include
     - pip install --upgrade pip
+
     # we will pay the ~13 minute cost of compiling Cython only when a new
     # version is scraped in by pip; otherwise, use the cached
     # wheel shippable places on Amazon S3 after we build it once

tools/openblas_support.py

@@ -1,5 +1,141 @@
+from __future__ import division, absolute_import, print_function
 import os
+import sys
 import textwrap
+import platform
+try:
+    from urllib.request import urlopen
+    from urllib.error import HTTPError
+except:
+    #Python2
+    from urllib2 import urlopen, HTTPError
+
+from tempfile import mkstemp, gettempdir
+import zipfile
+import tarfile
+
+OPENBLAS_V = 'v0.3.5'
+OPENBLAS_LONG = 'v0.3.5-274-g6a8b4269'
+BASE_LOC = ''
+RACKSPACE = 'https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com'
+ARCHITECTURES = ['', 'windows', 'darwin', 'arm', 'x86', 'ppc64']
+
+IS_32BIT = sys.maxsize < 2**32
+def get_arch():
+    if platform.system() == 'Windows':
+        ret = 'windows'
+    elif platform.system() == 'Darwin':
+        ret = 'darwin'
+    # Python3 returns a named tuple, but Python2 does not, so we are stuck
+    elif 'arm' in os.uname()[-1]:
+        ret = 'arm';
+    elif 'aarch64' in os.uname()[-1]:
+        ret = 'arm';
+    elif 'x86' in os.uname()[-1]:
+        ret = 'x86'
+    elif 'ppc64' in os.uname()[-1]:
+        ret = 'ppc64'
+    else:
+        ret = ''
+    assert ret in ARCHITECTURES
+    return ret
+
+def download_openblas(target, arch):
+    filename = ''
+    if arch == 'arm':
+        # ARMv8 OpenBLAS built using script available here:
+        # https://github.com/tylerjereddy/openblas-static-gcc/tree/master/ARMv8
+        # build done on GCC compile farm machine named gcc115
+        # tarball uploaded manually to an unshared Dropbox location
+        filename = ('https://www.dropbox.com/s/pbqkxzlmih4cky1/'
+                    'openblas-{}-armv8.tar.gz?dl=1'.format(OPENBLAS_V))
+        typ = 'tar.gz'
+    elif arch == 'ppc64':
+        # build script for POWER8 OpenBLAS available here:
+        # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8
+        # built on GCC compile farm machine named gcc112
+        # manually uploaded tarball to an unshared Dropbox location
+        filename = ('https://www.dropbox.com/s/zcwhk7c2zptwy0s/'
+                    'openblas-{}-ppc64le-power8.tar.gz?dl=1'.format(OPENBLAS_V))
+        typ = 'tar.gz'
+    elif arch == 'darwin':
+        filename = '{0}/openblas-{1}-macosx_10_9_x86_64-gf_1becaaa.tar.gz'.format(
+                        RACKSPACE, OPENBLAS_LONG)
+        typ = 'tar.gz'
+    elif arch == 'windows':
+        if IS_32BIT:
+            suffix = 'win32-gcc_7_1_0.zip'
+        else:
+            suffix = 'win_amd64-gcc_7_1_0.zip'
+        filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix)
+        typ = 'zip'
+    elif arch == 'x86':
+        if IS_32BIT:
+            suffix = 'manylinux1_i686.tar.gz'
+        else:
+            suffix = 'manylinux1_x86_64.tar.gz'
+        filename = '{0}/openblas-{1}-{2}'.format(RACKSPACE, OPENBLAS_LONG, suffix)
+        typ = 'tar.gz'
+    if not filename:
+        return None
+    try:
+        with open(target, 'wb') as fid:
+            fid.write(urlopen(filename).read())
+    except HTTPError:
+        print('Could not download "%s"' % filename)
+        return None
+    return typ
+
+def setup_openblas(arch=get_arch()):
+    '''
+    Download and setup an openblas library for building. If successful,
+    the configuration script will find it automatically.
+
+    Returns
+    -------
+    msg : str
+        path to extracted files on success, otherwise indicates what went wrong
+        To determine success, do ``os.path.exists(msg)``
+    '''
+    _, tmp = mkstemp()
+    if not arch:
+        raise ValueError('unknown architecture')
+    typ = download_openblas(tmp, arch)
+    if not typ:
+        return ''
+    if arch == 'windows':
+        if not typ == 'zip':
+            return 'expecting to download zipfile on windows, not %s' % str(typ)
+        return unpack_windows_zip(tmp)
+    else:
+        if not typ == 'tar.gz':
+            return 'expecting to download tar.gz, not %s' % str(typ)
+        return unpack_targz(tmp)
+
+def unpack_windows_zip(fname):
+    import sysconfig
+    with zipfile.ZipFile(fname, 'r') as zf:
+        # Get the openblas.a file, but not openblas.dll.a nor openblas.dev.a
+        lib = [x for x in zf.namelist() if OPENBLAS_LONG in x and
+                  x.endswith('a') and not x.endswith('dll.a') and
+                  not x.endswith('dev.a')]
+        if not lib:
+            return 'could not find libopenblas_%s*.a ' \
+                    'in downloaded zipfile' % OPENBLAS_LONG
+        target = os.path.join(gettempdir(), 'openblas.a')
+        with open(target, 'wb') as fid:
+            fid.write(zf.read(lib[0]))
+    return target
+
+def unpack_targz(fname):
+    target = os.path.join(gettempdir(), 'openblas')
+    if not os.path.exists(target):
+        os.mkdir(target)
+    with tarfile.open(fname, 'r') as zf:
+        # TODO: check that all the zf.getnames() files do not escape the
+        # extract directory (no leading '../', '/')
+        zf.extractall(target)
+    return target
 
 def make_init(dirname):
     '''
@@ -40,3 +176,34 @@ def make_init(dirname):
                               stacklevel=1)
     """))
 
+def test_setup(arches):
+    '''
+    Make sure all the downloadable files exist and can be opened
+    '''
+    for arch in arches:
+        if arch == '':
+            continue
+        try:
+            target = setup_openblas(arch)
+        except:
+            print('Could not setup %s' % arch)
+            raise
+        if not target:
+            raise RuntimeError('Could not setup %s' % arch)
+        print(target)
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(
+        description='Download and expand an OpenBLAS archive for this ' \
+                    'architecture')
+    parser.add_argument('--test', nargs='*', default=None,
+        help='Test different architectures. "all", or any of %s' % ARCHITECTURES)
+    args = parser.parse_args()
+    if args.test is None:
+        print(setup_openblas())
+    else:
+        if len(args.test) == 0 or 'all' in args.test:
+            test_setup(ARCHITECTURES)
+        else:
+            test_setup(args.test)

tools/pypy-test.sh

@@ -11,22 +11,19 @@ sudo apt-get -yq install libatlas-base-dev liblapack-dev gfortran-5
 F77=gfortran-5 F90=gfortran-5 \
 
 # Download the proper OpenBLAS x64 precompiled library
-OPENBLAS=openblas-v0.3.5-274-g6a8b4269-manylinux1_x86_64.tar.gz
-echo getting $OPENBLAS
-wget -q https://3f23b170c54c2533c070-1c8a9b3114517dc5fe17b7c3f8c63a43.ssl.cf2.rackcdn.com/$OPENBLAS -O openblas.tar.gz
-mkdir -p openblas
-(cd openblas; tar -xf ../openblas.tar.gz)
-export LD_LIBRARY_PATH=$PWD/openblas/usr/local/lib
-export LIB=$PWD/openblas/usr/local/lib
-export INCLUDE=$PWD/openblas/usr/local/include
+target=$(python tools/openblas_support.py)
+echo getting OpenBLAS into $target
+export LD_LIBRARY_PATH=$target/usr/local/lib
+export LIB=$target/usr/local/lib
+export INCLUDE=$target/usr/local/include
 
 # Use a site.cfg to build with local openblas
 cat << EOF > site.cfg
 [openblas]
 libraries = openblas
-library_dirs = $PWD/openblas/usr/local/lib:$LIB
-include_dirs = $PWD/openblas/usr/local/lib:$LIB
-runtime_library_dirs = $PWD/openblas/usr/local/lib
+library_dirs = $target/usr/local/lib:$LIB
+include_dirs = $target/usr/local/lib:$LIB
+runtime_library_dirs = $target/usr/local/lib
 EOF
 
 echo getting PyPy 3.6 nightly

tools/travis-before-install.sh

@@ -26,14 +26,9 @@ if [ -n "$INSTALL_PICKLE5" ]; then
 fi
 
 if [ -n "$PPC64_LE" ]; then
-  # build script for POWER8 OpenBLAS available here:
-  # https://github.com/tylerjereddy/openblas-static-gcc/blob/master/power8
-  # built on GCC compile farm machine named gcc112
-  # manually uploaded tarball to an unshared Dropbox location
-  wget -O openblas-power8.tar.gz https://www.dropbox.com/s/zcwhk7c2zptwy0s/openblas-v0.3.5-ppc64le-power8.tar.gz?dl=0
-  tar zxvf openblas-power8.tar.gz
-  sudo cp -r ./64/lib/* /usr/lib
-  sudo cp ./64/include/* /usr/include
+  target=$(python tools/openblas_support.py)
+  sudo cp -r $target/64/lib/* /usr/lib
+  sudo cp $target/64/include/* /usr/include
 fi
 
 pip install --upgrade pip setuptools