MRG: Minor fixes to testing framework (mne-tools#4541)

* FIX: Minor fixes to testing framework

* FIX: Fix AppVeyor

* FIX: Botched rebase

.travis.yml

@@ -13,78 +13,78 @@ matrix:
     include:
         # Full (Linux, 2.7) first half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
-               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi nose pytest pytest-cov"
+               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
                OPTION=""
         # Full (Linux, 2.7) second half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
-               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi nose pytest pytest-cov"
+               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
                OPTION=""
 
         # OSX first half
         - os: osx
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
-               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi nose pytest pytest-cov"
+               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
                OPTION=""
         # OSX second half
         - os: osx
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
-               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi nose pytest pytest-cov"
+               PIP_DEPENDENCIES="git+git://github.com/nipy/PySurfer.git nitime faulthandler joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
                OPTION=""
 
         # Py3k + non-default stim channel first half
         - os: linux
           env: PYTHON_VERSION=3.6 TEST_LOCATION=install MNE_STIM_CHANNEL=STI101
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest"
-               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer pytest-sugar pytest-cov pytest-attrib"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest pytest-cov"
+               PIP_DEPENDENCIES="nitime joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
                OPTION=""
         # Py3k + non-default stim channel second half
         - os: linux
           env: PYTHON_VERSION=3.6 TEST_LOCATION=install MNE_STIM_CHANNEL=STI101
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest"
-               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer pytest-sugar pytest-cov pytest-attrib"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest pytest-cov"
+               PIP_DEPENDENCIES="nitime joblib nibabel codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
                OPTION=""
 
         # Old dependencies first half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 pytest"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 nose pytest pytest-cov"
+               PIP_DEPENDENCIES="codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
                OPTION=""
         # Old dependencies second half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 pytest"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 nose pytest pytest-cov"
+               PIP_DEPENDENCIES="codecov pytest-sugar pytest-faulthandler"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
                OPTION=""
 
         # Minimal first half
         - os: linux
           env: DEPS=minimial
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pytest"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib nose pytest pytest-cov"
+               PIP_DEPENDENCIES="codecov faulthandler pytest-sugar pytest-faulthandler"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
                OPTION=""
         # Minimal second half
         - os: linux
           env: DEPS=minimial
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pytest"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib nose pytest pytest-cov"
+               PIP_DEPENDENCIES="codecov faulthandler pytest-sugar pytest-faulthandler"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
                OPTION=""
 
         # No data + style testing
         - os: linux
           env: DEPS=nodata MNE_DONTWRITE_HOME=true MNE_FORCE_SERIAL=true MNE_SKIP_NETWORK_TEST=1
-               CONDA_DEPENDENCIES="numpy scipy matplotlib sphinx pytest"
-               PIP_DEPENDENCIES="flake8 numpydoc codespell git+git://github.com/PyCQA/pydocstyle.git codecov nose-timer nose-faulthandler check-manifest pytest-sugar pytest-cov pytest-attrib"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib sphinx nose pytest pytest-cov"
+               PIP_DEPENDENCIES="flake8 numpydoc codespell git+git://github.com/PyCQA/pydocstyle.git codecov check-manifest pytest-sugar"
                MNE_DIRS=""
                OPTION="--doctest-ignore-import-errors"
 
@@ -162,6 +162,7 @@ script:
       else
         CONDITION='not ultra_slow_test';
       fi;
+    - python -c "import mne; print(mne.sys_info())"
     # Determine directories to test
     - if [ "${MNE_DIRS}" != "" ]; then
         USE_DIRS="";
@@ -171,17 +172,14 @@ script:
       else
         USE_DIRS="mne/";
       fi;
-    - echo py.test --doctest-modules -a "\"${CONDITION}\"" ${OPTION} --cov=mne --showlocals --durations=20 ${USE_DIRS};
-    - py.test --doctest-modules -a "\"${CONDITION}\"" ${OPTION} --cov=mne --showlocals --durations=20 ${USE_DIRS};
+    - echo py.test -m "\"${CONDITION}\"" ${OPTION} ${USE_DIRS};
+    - py.test -m "\"${CONDITION}\"" ${OPTION} ${USE_DIRS};
     - if [ "${DEPS}" == "nodata" ]; then
         make pep;
         check-manifest --ignore doc,logo,mne/io/*/tests/data*,mne/io/tests/data,mne/preprocessing/tests/data;
       fi;
 
 after_success:
     # Need to run from source dir to exectue "git" commands
-    - if [ "${TEST_LOCATION}" == "src" ]; then
-        echo "Running codecov";
-        cd ${SRC_DIR};
-        codecov;
-      fi;
+    - cd ${SRC_DIR};
+    - codecov;

Makefile

@@ -39,15 +39,15 @@ testing_data:
 
 test: in
 	rm -f .coverage
-	$(PYTESTS) -a 'not ultra_slow_test' mne
+	$(PYTESTS) -m 'not ultraslowtest' mne
 
 test-verbose: in
 	rm -f .coverage
-	$(PYTESTS) -a 'not ultra_slow_test' mne --verbose
+	$(PYTESTS) -m 'not ultraslowtest' mne --verbose
 
 test-fast: in
 	rm -f .coverage
-	$(PYTESTS) -a 'not slow_test' mne
+	$(PYTESTS) -m 'not slowtest' mne
 
 test-full: in
 	rm -f .coverage

appveyor.yml

@@ -1,8 +1,8 @@
 environment:
   global:
       PYTHON: "C:\\conda"
-      CONDA_DEPENDENCIES: "setuptools numpy scipy matplotlib scikit-learn mayavi pandas h5py PIL pyside pytest"
-      PIP_DEPENDENCIES: "git+git://github.com/nipy/PySurfer.git nibabel nitime nose-timer nose-faulthandler pytest-attrib"
+      CONDA_DEPENDENCIES: "setuptools numpy scipy matplotlib scikit-learn mayavi pandas h5py PIL pyside nose pytest pytest-cov"
+      PIP_DEPENDENCIES: "git+git://github.com/nipy/PySurfer.git nibabel nitime codecov"
   matrix:
       - PYTHON_VERSION: "2.7"
         PYTHON_ARCH: "64"
@@ -13,7 +13,7 @@ install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "activate test"
   - "python setup.py develop"
-  - "SET MNE_SKIP_NETWORK_TESTS=1"
+  - "python -c \"import mne; print(mne.sys_info())\""
   - "SET MNE_FORCE_SERIAL=true"  # otherwise joblib will bomb
   - "SET MNE_LOGGING_LEVEL=warning"
   - "python -c \"import mne; mne.datasets.testing.data_path()\""
@@ -22,4 +22,7 @@ build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Run the project tests, but (sadly) exclude ones that take a long time
-  - "py.test --doctest-modules -a \"not slow_test\" --showlocals --durations=20 mne"
+  - "py.test -m \"not ultraslowtest\" mne "
+
+on_success:
+  - "codecov"

doc/advanced_setup.rst

@@ -64,7 +64,7 @@ You can test if MNE CUDA support is working by running the associated test:
 
 .. code-block:: bash
 
-    $ nosetests mne/tests/test_filter.py
+    $ pytest mne/tests/test_filter.py
 
 If ``MNE_USE_CUDA=true`` and all tests pass with none skipped, then
 MNE-Python CUDA support works.

doc/configure_git.rst

@@ -152,7 +152,7 @@ These steps can be broken out to be more explicit as:
 
 #. Ensure unit tests pass
 
-   Make sure before starting to code that all unit tests pass with `nose`_:
+   Make sure before starting to code that all unit tests pass with `pytest`_:
 
    .. code-block:: bash
 

doc/contributing.rst

@@ -55,7 +55,7 @@ Code guidelines
 
   .. code-block:: bash
 
-     $ nosetests mne/tests/test_evoked.py:test_io_evoked -x --verbose
+     $ pytest mne/tests/test_evoked.py:test_io_evoked -x --verbose
 
   Make sure you have the testing dataset, which you can get by doing::
 
@@ -70,7 +70,7 @@ Pull requests
 ^^^^^^^^^^^^^
 * Address one issue per pull request (PR).
 * Avoid unnecessary cosmetic changes in PRs.
-* Minimize test timing while maximizing coverage. Use ``nosetests --with-timer`` on modified tests.
+* Minimize test timing while maximizing coverage. Use ``pytest --durations=20`` on modified tests.
 * Update the ``doc/whats_new.rst`` file last, just before merge to avoid merge conflicts.
 
 Naming

doc/faq.rst

@@ -81,7 +81,6 @@ reporting a bug, which should look something like this::
     nibabel:       Not found
     nitime:        Not found
     mayavi:        Not found
-    nose:          1.3.7
     pandas:        Not found
     pycuda:        Not found
     skcuda:        Not found

doc/git_links.inc

@@ -62,8 +62,6 @@
 .. _pep8: https://pypi.python.org/pypi/pep8
 .. _pyflakes: https://pypi.python.org/pypi/pyflakes
 .. _coverage: https://pypi.python.org/pypi/coverage
-.. _nose-timer: https://pypi.python.org/pypi/nose-timer
-.. _nosetests: https://nose.readthedocs.org/en/latest/
 .. _mayavi: http://mayavi.sourceforge.net/
 .. _nitime: http://nipy.org/nitime/
 .. _joblib: https://pypi.python.org/pypi/joblib

doc/known_projects.inc

@@ -50,8 +50,8 @@
 .. _Astropy: http://www.astropy.org
 .. _`Astropy GitHub`: http://github.com/astropy/astropy
 
-.. Nose
-.. _Nose: https://nose.readthedocs.io/
+.. Pytest
+.. _pytest: https://docs.pytest.org/
 
 .. Flake8
 .. _Flake8: http://flake8.pycqa.org/

mne/beamformer/tests/test_lcmv.py

@@ -1,7 +1,7 @@
 from copy import deepcopy
 import os.path as op
 
-
+import pytest
 from nose.tools import assert_true, assert_raises
 import numpy as np
 from numpy.testing import (assert_array_almost_equal, assert_array_equal,
@@ -15,7 +15,7 @@
                             lcmv_epochs, lcmv_raw, tf_lcmv)
 from mne.beamformer._lcmv import _lcmv_source_power, _reg_pinv
 from mne.externals.six import advance_iterator
-from mne.utils import run_tests_if_main, slow_test
+from mne.utils import run_tests_if_main
 
 
 data_path = testing.data_path(download=False)
@@ -97,7 +97,7 @@ def _get_data(tmin=-0.1, tmax=0.15, all_forward=True, epochs=True,
         forward_surf_ori, forward_fixed, forward_vol
 
 
-@slow_test
+@pytest.mark.slowtest
 @testing.requires_testing_data
 def test_lcmv():
     """Test LCMV with evoked data and single trials."""

mne/channels/tests/test_interpolation.py

@@ -3,11 +3,12 @@
 
 import numpy as np
 from numpy.testing import (assert_allclose, assert_array_equal)
+import pytest
 from nose.tools import assert_raises, assert_equal, assert_true
 
 from mne import io, pick_types, pick_channels, read_events, Epochs
 from mne.channels.interpolation import _make_interpolation_matrix
-from mne.utils import run_tests_if_main, slow_test
+from mne.utils import run_tests_if_main
 
 base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
 raw_fname = op.join(base_dir, 'test_raw.fif')
@@ -41,7 +42,7 @@ def _load_data():
     return raw, epochs, epochs_eeg, epochs_meg
 
 
-@slow_test
+@pytest.mark.slowtest
 def test_interpolation():
     """Test interpolation"""
     raw, epochs, epochs_eeg, epochs_meg = _load_data()

mne/commands/tests/test_commands.py

@@ -4,6 +4,7 @@
 import shutil
 import glob
 import warnings
+import pytest
 from nose.tools import assert_true, assert_raises
 from numpy.testing import assert_equal, assert_allclose
 
@@ -19,7 +20,7 @@
 from mne.io import read_raw_fif
 from mne.utils import (run_tests_if_main, _TempDir, requires_mne, requires_PIL,
                        requires_mayavi, requires_tvtk, requires_freesurfer,
-                       ArgvSetter, slow_test, ultra_slow_test)
+                       ArgvSetter)
 
 
 base_dir = op.join(op.dirname(__file__), '..', '..', 'io', 'tests', 'data')
@@ -41,7 +42,7 @@ def check_usage(module, force_help=False):
         assert_true('Usage: ' in out.stdout.getvalue())
 
 
-@slow_test
+@pytest.mark.slowtest
 def test_browse_raw():
     """Test mne browse_raw."""
     check_usage(mne_browse_raw)
@@ -84,7 +85,7 @@ def test_clean_eog_ecg():
     assert_true(len(fnames) == 3)  # raw plus two projs
 
 
-@slow_test
+@pytest.mark.slowtest
 def test_compute_proj_ecg_eog():
     """Test mne compute_proj_ecg/eog."""
     for fun in (mne_compute_proj_ecg, mne_compute_proj_eog):
@@ -177,7 +178,7 @@ def test_maxfilter():
             assert_true(check in out.stdout.getvalue(), check)
 
 
-@slow_test
+@pytest.mark.slowtest
 @requires_mayavi
 @requires_PIL
 @testing.requires_testing_data
@@ -199,7 +200,8 @@ def test_surf2bem():
     check_usage(mne_surf2bem)
 
 
-@ultra_slow_test
+@pytest.mark.slowtest
+@pytest.mark.ultraslowtest
 @requires_freesurfer
 @testing.requires_testing_data
 def test_watershed_bem():
@@ -223,7 +225,8 @@ def test_watershed_bem():
         mne_watershed_bem.run()
 
 
-@ultra_slow_test
+@pytest.mark.slowtest
+@pytest.mark.ultraslowtest
 @requires_freesurfer
 @sample.requires_sample_data
 def test_flash_bem():

mne/connectivity/tests/test_spectral.py

@@ -2,13 +2,14 @@
 
 import numpy as np
 from numpy.testing import assert_array_almost_equal
+import pytest
 from nose.tools import assert_true, assert_raises
 
 from mne.connectivity import spectral_connectivity
 from mne.connectivity.spectral import _CohEst, _get_n_epochs
 
 from mne import SourceEstimate
-from mne.utils import run_tests_if_main, slow_test
+from mne.utils import run_tests_if_main
 from mne.filter import filter_data
 
 warnings.simplefilter('always')
@@ -30,7 +31,7 @@ def _stc_gen(data, sfreq, tmin, combo=False):
             yield (arr, stc)
 
 
-@slow_test
+@pytest.mark.slowtest
 def test_spectral_connectivity():
     """Test frequency-domain connectivity methods"""
     # Use a case known to have no spurious correlations (it would bad if

mne/coreg.py

@@ -1150,7 +1150,7 @@ def scale_source_space(subject_to, src_name, subject_from=None, scale=None,
         spacing = src_name  # spacing in mm
         src_pattern = src_fname
     else:
-        match = re.match("(oct|ico)-?(\d+)$", src_name)
+        match = re.match(r"(oct|ico)-?(\d+)$", src_name)
         if match:
             spacing = '-'.join(match.groups())
             src_pattern = src_fname