Merge pull request numpy#14378 from mattip/test-dependencies

BUILD: move all test dependencies to ./test_requirements.txt

.appveyor.yml

@@ -34,13 +34,11 @@ environment:
       PYTHON_VERSION: 3.6
       PYTHON_ARCH: 64
       TEST_MODE: full
-      INSTALL_PICKLE5: 1
 
     - PYTHON: C:\Python37-x64
       PYTHON_VERSION: 3.7
       PYTHON_ARCH: 64
       TEST_MODE: full
-      INSTALL_PICKLE5: 1
 
 init:
   - "ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH%"
@@ -98,10 +96,8 @@ install:
   # Upgrade to the latest pip.
   - 'python -m pip install -U pip setuptools wheel'
 
-  - if [%INSTALL_PICKLE5%]==[1] echo pickle5 >> tools/ci/appveyor/requirements.txt
-
   # Install the numpy test dependencies.
-  - 'pip install -U --timeout 5 --retries 2 -r tools/ci/appveyor/requirements.txt'
+  - 'pip install -U --timeout 5 --retries 2 -r test_requirements.txt'
 
 build_script:
   # Here, we add MinGW to the path to be able to link an OpenBLAS.dll

azure-pipelines.yml

@@ -21,11 +21,12 @@ jobs:
            apt-get -y update && \
            apt-get -y install python3.6-dev python3-pip locales python3-certifi && \
            locale-gen fr_FR && update-locale && \
-           pip3 install setuptools nose cython==0.29.0 pytest pytz pickle5 && \
            apt-get -y install gfortran-5 wget && \
            target=\$(python3 tools/openblas_support.py) && \
            cp -r \$target/usr/local/lib/* /usr/lib && \
            cp \$target/usr/local/include/* /usr/include && \
+           python3 -m pip install --user --upgrade pip setuptools && \
+           python3 -m pip install --user -r test_requirements.txt && \
            python3 -m pip install . && \
            F77=gfortran-5 F90=gfortran-5 \
            CFLAGS='-UNDEBUG -std=c99' python3 runtests.py -n --mode=full -- -rsx --junitxml=junit/test-results.xml && \
@@ -85,7 +86,9 @@ jobs:
     displayName: 'install pre-built openblas'
   - script: python -m pip install --upgrade pip setuptools wheel
     displayName: 'Install tools'
-  - script: python -m pip install cython nose pytz pytest pickle5 vulture docutils sphinx==1.8.5 numpydoc
+  - script: |
+      python -m pip install -r test_requirements.txt
+      python -m pip install vulture docutils sphinx==1.8.5 numpydoc
     displayName: 'Install dependencies; some are optional to avoid test skips'
   - script: /bin/bash -c "! vulture . --min-confidence 100 --exclude doc/,numpy/distutils/ | grep 'unreachable'"
     displayName: 'Check for unreachable code paths in Python modules'
@@ -140,13 +143,11 @@ jobs:
           PYTHON_VERSION: '3.6'
           PYTHON_ARCH: 'x64'
           TEST_MODE: full
-          INSTALL_PICKLE5: 1
           BITS: 64
         Python37-64bit-full:
           PYTHON_VERSION: '3.7'
           PYTHON_ARCH: 'x64'
           TEST_MODE: full
-          INSTALL_PICKLE5: 1
           BITS: 64
   steps:
   - task: UsePythonVersion@0
@@ -156,11 +157,8 @@ jobs:
       architecture: $(PYTHON_ARCH)
   - script: python -m pip install --upgrade pip setuptools wheel
     displayName: 'Install tools'
-  - script: python -m pip install cython nose pytz pytest
+  - script: python -m pip install -r test_requirements.txt
     displayName: 'Install dependencies; some are optional to avoid test skips'
-  - script: if [%INSTALL_PICKLE5%]==[1] python -m pip install pickle5
-    displayName: 'Install optional pickle5 backport (only for python3.6 and 3.7)'
-
   - powershell: |
       $pyversion = python -c "from __future__ import print_function; import sys; print(sys.version.split()[0])"
       Write-Host "Python Version: $pyversion"
@@ -220,3 +218,20 @@ jobs:
       testResultsFiles: '**/test-*.xml'
       testRunTitle: 'Publish test results for PyPy3'
       failTaskOnFailedTests: true
+
+- job: Check_dependencies
+  pool:
+    vmIMage: 'ubuntu-16.04'
+  steps:
+  - script: |
+        set -e
+        sudo apt-get install python3-venv
+        python3 -mvenv testenv
+        source testenv/bin/activate
+        python -m pip install --upgrade pip setuptools
+        python -m pip list -o > /tmp/current_pkgs
+        python -m pip install -r test_requirements.txt
+        python -m pip list -o > /tmp/test_pkgs
+        # any newer packages should cause the diff to return 1, which fails the job
+        diff /tmp/current_pkgs /tmp/test_pkgs
+    displayName: 'Show any possible updates to packages'

test_requirements.txt

@@ -0,0 +1,7 @@
+cython==0.29.13
+pytest==5.1.1
+pytz==2019.2
+pytest-cov==2.7.1
+pickle5; python_version == '3.7'
+pickle5; python_version == '3.6' and platform_python_implementation != 'PyPy'
+nose

tools/pypy-test.sh

@@ -32,7 +32,7 @@ mkdir -p pypy3
 (cd pypy3; tar --strip-components=1 -xf ../pypy.tar.bz2)
 pypy3/bin/pypy3 -mensurepip
 pypy3/bin/pypy3 -m pip install --upgrade pip setuptools
-pypy3/bin/pypy3 -m pip install --user cython==0.29.0 pytest pytz --no-warn-script-location
+pypy3/bin/pypy3 -m pip install --user -r test_requirements.txt --no-warn-script-location
 
 echo
 echo pypy3 version 

tools/travis-before-install.sh

@@ -30,12 +30,8 @@ fi
 source venv/bin/activate
 python -V
 
-if [ -n "$INSTALL_PICKLE5" ]; then
-  pip install pickle5
-fi
-
+popd
 
 pip install --upgrade pip setuptools
-pip install pytz cython pytest
+pip install -r test_requirements.txt
 if [ -n "$USE_ASV" ]; then pip install asv; fi
-popd

tools/travis-test.sh

@@ -65,12 +65,12 @@ setup_base()
 
 run_test()
 {
+  $PIP install -r test_requirements.txt
   if [ -n "$USE_DEBUG" ]; then
     export PYTHONPATH=$PWD
   fi
 
   if [ -n "$RUN_COVERAGE" ]; then
-    $PIP install pytest-cov
     COVERAGE_FLAG=--coverage
   fi
 
@@ -158,14 +158,10 @@ if [ -n "$USE_WHEEL" ] && [ $# -eq 0 ]; then
   # Move out of source directory to avoid finding local numpy
   pushd dist
   $PIP install --pre --no-index --upgrade --find-links=. numpy
-  $PIP install nose pytest
-
-  if [ -n "$INSTALL_PICKLE5" ]; then
-    $PIP install pickle5
-  fi
-
   popd
+
   run_test
+
 elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then
   # use an up-to-date pip / setuptools inside the venv
   $PIP install -U virtualenv
@@ -182,11 +178,6 @@ elif [ -n "$USE_SDIST" ] && [ $# -eq 0 ]; then
   # Move out of source directory to avoid finding local numpy
   pushd dist
   $PIP install numpy*
-  $PIP install nose pytest
-  if [ -n "$INSTALL_PICKLE5" ]; then
-    $PIP install pickle5
-  fi
-
   popd
   run_test
 else