CI: Run benchmarks (scipy#8779)

.circleci/config.yml

@@ -1,13 +1,23 @@
 version: 2
-jobs:
-  build_docs:
-    docker:
-      # CircleCI maintains a library of pre-built images
-      # documented at https://circleci.com/docs/2.0/circleci-images/
-      - image: circleci/python:3.7.0
 
-    working_directory: ~/repo
+# Aliases to reuse
+_defaults: &defaults
+  docker:
+    # CircleCI maintains a library of pre-built images
+    # documented at https://circleci.com/docs/2.0/circleci-images/
+    - image: circleci/python:3.7.0
+  working_directory: ~/repo
+
+_debian: &debian
+  name: install Debian dependencies
+  command: |
+    sudo apt-get update
+    sudo apt-get install libatlas-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libfreetype6-dev libpng-dev zlib1g zlib1g-dev texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex fonts-freefont-otf
 
+jobs:
+# Build SciPy from source
+  build_scipy:
+    <<: *defaults
     steps:
       - checkout
 
@@ -30,10 +40,7 @@ jobs:
             git submodule update
 
       - run:
-          name: install Debian dependencies
-          command: |
-            sudo apt-get update
-            sudo apt-get install libatlas-dev libatlas-base-dev liblapack-dev gfortran libgmp-dev libmpfr-dev libfreetype6-dev libpng-dev zlib1g zlib1g-dev texlive-fonts-recommended texlive-latex-recommended texlive-latex-extra texlive-generic-extra latexmk texlive-xetex fonts-freefont-otf
+          <<: *debian
 
       - run:
           name: setup Python venv
@@ -42,9 +49,31 @@ jobs:
             . venv/bin/activate
             pip install --install-option="--no-cython-compile" cython
             pip install numpy
-            pip install nose mpmath argparse Pillow codecov matplotlib "Sphinx!=3.1.0"
+            pip install nose mpmath argparse Pillow codecov matplotlib "Sphinx!=3.1.0" asv
             pip install pybind11
 
+      - run:
+          name: build SciPy
+          command: |
+            . venv/bin/activate
+            export SHELL=$(which bash)
+            python -u runtests.py -gbj2
+
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - repo
+
+# Build docs
+  build_docs:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+
+      - run:
+          <<: *debian
+
       - run:
           name: build docs
           command: |
@@ -67,18 +96,41 @@ jobs:
           paths:
             - html-scipyorg
 
+# Run benchmarks
+
+  run_benchmarks:
+    <<: *defaults
+    steps:
+      - attach_workspace:
+          at: ~/
+
+      - run:
+          <<: *debian
+
+      - run:
+          name: run asv
+          command: |
+            . venv/bin/activate
+            pip install pyfftw cffi pytest
+            export PYTHONPATH=$PWD/build/testenv/lib/python3.7/site-packages
+            cd benchmarks
+            asv machine --machine CircleCI
+            export SCIPY_GLOBAL_BENCH_NUMTRIALS=1
+            export SCIPY_ALLOW_BENCH_IMPORT_ERRORS=0
+            time asv --config asv.conf.json dev -m CircleCI --python=same --bench '^((?!BenchGlobal|QuadraticAssignment).)*$'
+            asv --config asv.conf.json publish
+
+      - store_artifacts:
+          path: benchmarks/html
+          destination: html-benchmarks
 
 # Upload build output to scipy/devdocs repository, using SSH deploy keys.
 # The keys are only available for builds on master branch.
 # https://developer.github.com/guides/managing-deploy-keys/
 # https://circleci.com/docs/2.0/configuration-reference/#add_ssh_keys
 
   deploy:
-    docker:
-      - image: circleci/python:3.7.0
-
-    working_directory: ~/repo
-
+    <<: *defaults
     steps:
       - attach_workspace:
           at: /tmp/build
@@ -112,11 +164,16 @@ workflows:
   version: 2
   default:
     jobs:
-      - build_docs
+      - build_scipy
+      - build_docs:
+          requires:
+            - build_scipy
+      - run_benchmarks:
+          requires:
+            - build_scipy
       - deploy:
           requires:
             - build_docs
-
           filters:
             branches:
               only: master

.travis.yml

@@ -277,9 +277,10 @@ script:
     if [ "${TRAVIS_PULL_REQUEST}" != "false" ] && [ "${REFGUIDE_CHECK}" == "1" ]; then
       python runtests.py -g --refguide-check
     fi
+  # pyfftw missing should raise an error when running benchmarks with SCIPY_ALLOW_BENCH_IMPORT_ERRORS=0
   - |
     if [ "${TRAVIS_PULL_REQUEST}" != "false" ] && [ "${ASV_CHECK}" == "1" ]; then
-      (cd benchmarks && python -masv check -E existing)
+      (cd benchmarks && python -masv check -E existing && ! SCIPY_ALLOW_BENCH_IMPORT_ERRORS=0 python -masv check -E existing > /dev/null)
     fi
   # Check dynamic symbol hiding works on Linux
   - |

benchmarks/benchmarks/blas_lapack.py

@@ -1,12 +1,9 @@
 import numpy as np
+from .common import Benchmark, safe_import
 
-try:
+with safe_import():
     import scipy.linalg.lapack as la
     import scipy.linalg.blas as bla
-except ImportError:
-    pass
-
-from .common import Benchmark
 
 
 class GetBlasLapackFuncs(Benchmark):

benchmarks/benchmarks/cluster.py

@@ -1,13 +1,11 @@
 import numpy as np
 from numpy.testing import suppress_warnings
 
-from .common import Benchmark
+from .common import Benchmark, safe_import
 
-try:
+with safe_import():
     from scipy.cluster.hierarchy import linkage
     from scipy.cluster.vq import kmeans, kmeans2, vq
-except ImportError:
-    pass
 
 
 class HierarchyLinkage(Benchmark):

benchmarks/benchmarks/common.py

@@ -142,3 +142,18 @@ def decorator(func):
             setattr(func, key, value)
         return func
     return decorator
+
+
+class safe_import(object):
+
+    def __enter__(self):
+        self.error = False
+        return self
+
+    def __exit__(self, type_, value, traceback):
+        if type_ is not None:
+            self.error = True
+            suppress = not (
+                os.getenv('SCIPY_ALLOW_BENCH_IMPORT_ERRORS', '1').lower() in
+                ('0', 'false') or not issubclass(type_, ImportError))
+            return suppress

benchmarks/benchmarks/cython_special.py

@@ -2,12 +2,10 @@
 import numpy as np
 from scipy import special
 
-try:
-    from scipy.special import cython_special
-except ImportError:
-    pass
+from .common import with_attributes, safe_import
 
-from .common import with_attributes
+with safe_import():
+    from scipy.special import cython_special
 
 
 FUNC_ARGS = {

benchmarks/benchmarks/fft_basic.py

@@ -8,24 +8,25 @@
 
 import scipy.fftpack
 import numpy.fft
-try:
+from .common import Benchmark, safe_import
+
+with safe_import() as exc:
     import scipy.fft as scipy_fft
     has_scipy_fft = True
-except ImportError:
-    scipy_fft = {}
+if exc.error:
     has_scipy_fft = False
 
-from .common import Benchmark
 
-try:
+with safe_import() as exc:
     import pyfftw.interfaces.numpy_fft as pyfftw_fft
     import pyfftw
     pyfftw.interfaces.cache.enable()
     has_pyfftw = True
-except ImportError:
+if exc.error:
     pyfftw_fft = {}
     has_pyfftw = False
 
+
 class PyfftwBackend:
     """Backend for pyfftw"""
     __ua_domain__ = 'numpy.scipy.fft'

benchmarks/benchmarks/fftpack_pseudo_diffs.py

@@ -1,13 +1,10 @@
 """ Benchmark functions for fftpack.pseudo_diffs module
 """
 from numpy import arange, sin, cos, pi, exp, tanh, sign
+from .common import Benchmark, safe_import
 
-try:
+with safe_import():
     from scipy.fftpack import diff, fft, ifft, tilbert, hilbert, shift, fftfreq
-except ImportError:
-    pass
-
-from .common import Benchmark
 
 
 def direct_diff(x, k=1, period=None):

benchmarks/benchmarks/go_benchmark_functions/go_benchmark.py

@@ -2,10 +2,10 @@
 import numpy as np
 from numpy import abs, asarray
 
-try:
+from ..common import safe_import
+
+with safe_import():
     from scipy.special import factorial
-except ImportError:
-    pass
 
 
 class Benchmark(object):
@@ -49,10 +49,10 @@ class Benchmark(object):
     def __init__(self, dimensions):
         """
         Initialises the problem
-        
+
         Parameters
         ----------
-        
+
         dimensions : int
             The dimensionality of the problem
         """
@@ -172,9 +172,9 @@ def bounds(self):
 
     @property
     def N(self):
-        """        
+        """
         The dimensionality of the problem.
-        
+
         Returns
         -------
         N : int

benchmarks/benchmarks/go_benchmark_functions/go_funcs_M.py

@@ -1,11 +1,10 @@
 # -*- coding: utf-8 -*-
 from numpy import (abs, asarray, cos, exp, log, arange, pi, prod, sin, sqrt,
                    sum, tan)
-try:
+from .go_benchmark import Benchmark, safe_import
+
+with safe_import():
     from scipy.special import factorial
-except ImportError:
-    pass
-from .go_benchmark import Benchmark
 
 
 class Matyas(Benchmark):
@@ -208,7 +207,7 @@ class Mishra01(Benchmark):
 
        f_{\text{Mishra01}}(x) = (1 + x_n)^{x_n}
 
-     
+
     where
 
     .. math::
@@ -254,19 +253,19 @@ class Mishra02(Benchmark):
     .. math::
 
         f_{\text{Mishra02}}({x}) = (1 + x_n)^{x_n}
-     
+
 
      with
-     
+
      .. math::
-     
+
          x_n = n - \sum_{i=1}^{n-1} \frac{(x_i + x_{i+1})}{2}
 
 
-    Here, :math:`n` represents the number of dimensions and 
+    Here, :math:`n` represents the number of dimensions and
     :math:`x_i \in [0, 1]` for :math:`i = 1, ..., n`.
 
-    *Global optimum*: :math:`f(x) = 2` for :math:`x_i = 1` 
+    *Global optimum*: :math:`f(x) = 2` for :math:`x_i = 1`
     for :math:`i = 1, ..., n`
 
     .. [1] Jamil, M. & Yang, X.-S. A Literature Survey of Benchmark Functions
@@ -301,13 +300,13 @@ class Mishra03(Benchmark):
 
     .. math::
 
-       f_{\text{Mishra03}}(x) = \sqrt{\lvert \cos{\sqrt{\lvert x_1^2 
+       f_{\text{Mishra03}}(x) = \sqrt{\lvert \cos{\sqrt{\lvert x_1^2
        + x_2^2 \rvert}} \rvert} + 0.01(x_1 + x_2)
 
 
     with :math:`x_i \in [-10, 10]` for :math:`i = 1, 2`.
 
-    *Global optimum*: :math:`f(x) = -0.1999` for 
+    *Global optimum*: :math:`f(x) = -0.1999` for
     :math:`x = [-9.99378322, -9.99918927]`
 
     .. [1] Jamil, M. & Yang, X.-S. A Literature Survey of Benchmark Functions
@@ -473,7 +472,7 @@ class Mishra07(Benchmark):
     Here, :math:`n` represents the number of dimensions and
     :math:`x_i \in [-10, 10]` for :math:`i = 1, ..., n`.
 
-    *Global optimum*: :math:`f(x) = 0` for :math:`x_i = \sqrt{n}` 
+    *Global optimum*: :math:`f(x) = 0` for :math:`x_i = \sqrt{n}`
     for :math:`i = 1, ..., n`
 
     .. [1] Jamil, M. & Yang, X.-S. A Literature Survey of Benchmark Functions
@@ -608,7 +607,7 @@ class Mishra10(Benchmark):
 
     .. math::
     TODO - int(x) should be used instead of floor(x)!!!!!
-       f_{\text{Mishra10}}({x}) = \left[ \lfloor x_1 \perp x_2 \rfloor - 
+       f_{\text{Mishra10}}({x}) = \left[ \lfloor x_1 \perp x_2 \rfloor -
        \lfloor x_1 \rfloor - \lfloor x_2 \rfloor \right]^2
 
     with :math:`x_i \in [-10, 10]` for :math:`i =1, 2`.
@@ -691,7 +690,7 @@ class MultiModal(Benchmark):
 
     .. math::
 
-       f_{\text{MultiModal}}(x) = \left( \sum_{i=1}^n \lvert x_i \rvert 
+       f_{\text{MultiModal}}(x) = \left( \sum_{i=1}^n \lvert x_i \rvert
        \right) \left( \prod_{i=1}^n \lvert x_i \rvert \right)