[MRG+2] First iteration towards using pytest (mne-tools#4429)

* Move travis from nose-tests to py.test

* move other services rather than travis to pytest

* Start porting makefile

* echo locale in travis

* Add pytest appveyor

* Remove python 3.5

.travis.yml

@@ -1,4 +1,5 @@
 language: c
+dist: trusty
 sudo: false
 cache:
   apt: true
@@ -12,69 +13,80 @@ matrix:
     include:
         # Full (Linux, 2.7) first half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi"
-               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
+               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
+               OPTION=""
         # Full (Linux, 2.7) second half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi"
-               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
+               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
+               OPTION=""
 
         # OSX first half
         - os: osx
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi"
-               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
+               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
+               OPTION=""
         # OSX second half
         - os: osx
-          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi"
-               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels mayavi pytest"
+               PIP_DEPENDENCIES="pysurfer nitime faulthandler joblib nibabel codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
+               OPTION=""
 
         # Py3k + non-default stim channel first half
         - os: linux
           env: PYTHON_VERSION=3.6 TEST_LOCATION=install MNE_STIM_CHANNEL=STI101
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels"
-               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest"
+               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
+               OPTION=""
         # Py3k + non-default stim channel second half
         - os: linux
           env: PYTHON_VERSION=3.6 TEST_LOCATION=install MNE_STIM_CHANNEL=STI101
-               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels"
-               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pandas scikit-learn h5py pillow statsmodels nose pytest"
+               PIP_DEPENDENCIES="nitime joblib nibabel codecov nose-timer pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
+               OPTION=""
 
         # Old dependencies first half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 pytest"
+               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
+               OPTION=""
         # Old dependencies second half
         - os: linux
-          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler"
+          env: CONDA_DEPENDENCIES="numpy=1.8 scipy=0.12 matplotlib=1.3 pandas=0.12 scikit-learn=0.14 pytest"
+               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
+               OPTION=""
 
         # Minimal first half
         - os: linux
           env: DEPS=minimial
-               CONDA_DEPENDENCIES="numpy scipy matplotlib"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pytest"
+               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=". beamformer channels commands connectivity datasets decoding forward gui inverse_sparse io"
+               OPTION=""
         # Minimal second half
         - os: linux
           env: DEPS=minimial
-               CONDA_DEPENDENCIES="numpy scipy matplotlib"
-               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib pytest"
+               PIP_DEPENDENCIES="codecov nose-timer nose-faulthandler pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS="minimum_norm preprocessing realtime simulation stats time_frequency viz"
+               OPTION=""
 
         # No data + style testing
         - os: linux
           env: DEPS=nodata MNE_DONTWRITE_HOME=true MNE_FORCE_SERIAL=true MNE_SKIP_NETWORK_TEST=1
-               CONDA_DEPENDENCIES="numpy scipy matplotlib sphinx"
-               PIP_DEPENDENCIES="flake8 numpydoc codespell git+git://github.com/PyCQA/pydocstyle.git codecov nose-timer nose-faulthandler check-manifest"
+               CONDA_DEPENDENCIES="numpy scipy matplotlib sphinx pytest"
+               PIP_DEPENDENCIES="flake8 numpydoc codespell git+git://github.com/PyCQA/pydocstyle.git codecov nose-timer nose-faulthandler check-manifest pytest-sugar pytest-cov pytest-attrib"
                MNE_DIRS=""
+               OPTION="--doctest-ignore-import-errors"
 
 # Setup anaconda
 before_install:
@@ -141,11 +153,14 @@ install:
 
 
 script:
+    - echo "Print locale "
+    - locale
+    - echo "Other stuff"
     # OSX runs ~2x slower than Linux on Travis, so skip any slow ones there
     - if [ "${TRAVIS_OS_NAME}" == "osx" ]; then
-        CONDITION='!slow_test';
+        CONDITION='not slow_test';
       else
-        CONDITION='!ultra_slow_test';
+        CONDITION='not ultra_slow_test';
       fi;
     # Determine directories to test
     - if [ "${MNE_DIRS}" != "" ]; then
@@ -156,8 +171,8 @@ script:
       else
         USE_DIRS="mne/";
       fi;
-    - echo Testing with -a "${CONDITION}" ${COVERAGE} ${USE_DIRS}
-    - nosetests -a "${CONDITION}" --with-timer --timer-top-n 30 --timer-ok 5 --timer-warning 30 --verbosity=2 ${COVERAGE} ${USE_DIRS};
+    - echo py.test --doctest-modules -a "\"${CONDITION}\"" ${OPTION} --cov=mne --showlocals --durations=20 ${USE_DIRS};
+    - py.test --doctest-modules -a "\"${CONDITION}\"" ${OPTION} --cov=mne --showlocals --durations=20 ${USE_DIRS};
     - if [ "${DEPS}" == "nodata" ]; then
         make pep;
         check-manifest --ignore doc,logo,mne/io/*/tests/data*,mne/io/tests/data,mne/preprocessing/tests/data;

Makefile

@@ -3,7 +3,7 @@
 # caution: testing won't work on windows, see README
 
 PYTHON ?= python
-NOSETESTS ?= nosetests
+PYTESTS ?= py.test
 CTAGS ?= ctags
 CODESPELL_SKIPS ?= "*.fif,*.eve,*.gz,*.tgz,*.zip,*.mat,*.stc,*.label,*.w,*.bz2,*.annot,*.sulc,*.log,*.local-copy,*.orig_avg,*.inflated_avg,*.gii,*.pyc,*.doctree,*.pickle,*.inv,*.png,*.edf,*.touch,*.thickness,*.nofix,*.volume,*.defect_borders,*.mgh,lh.*,rh.*,COR-*,FreeSurferColorLUT.txt,*.examples,.xdebug_mris_calc,bad.segments,BadChannels,*.hist,empty_file,*.orig,*.js,*.map,*.ipynb,searchindex.dat"
 CODESPELL_DIRS ?= mne/ doc/ tutorials/ examples/
@@ -39,44 +39,41 @@ testing_data:
 
 test: in
 	rm -f .coverage
-	$(NOSETESTS) -a '!ultra_slow_test' mne
+	$(PYTESTS) -a 'not ultra_slow_test' mne
 
 test-verbose: in
 	rm -f .coverage
-	$(NOSETESTS) -a '!ultra_slow_test' mne --verbose
+	$(PYTESTS) -a 'not ultra_slow_test' mne --verbose
 
 test-fast: in
 	rm -f .coverage
-	$(NOSETESTS) -a '!slow_test' mne
+	$(PYTESTS) -a 'not slow_test' mne
 
 test-full: in
 	rm -f .coverage
-	$(NOSETESTS) mne
+	$(PYTESTS) mne
 
 test-no-network: in
-	sudo unshare -n -- sh -c 'MNE_SKIP_NETWORK_TESTS=1 nosetests mne'
+	sudo unshare -n -- sh -c 'MNE_SKIP_NETWORK_TESTS=1 py.test mne'
 
 test-no-testing-data: in
 	@MNE_SKIP_TESTING_DATASET_TESTS=true \
-	$(NOSETESTS) mne
+	$(PYTESTS) mne
 
 test-no-sample-with-coverage: in testing_data
 	rm -rf coverage .coverage
-	$(NOSETESTS) --with-coverage --cover-package=mne --cover-html --cover-html-dir=coverage
+	$(PYTESTS) --cov=mne --cov-report html:coverage
 
 test-doc: sample_data testing_data
-	$(NOSETESTS) --with-doctest --doctest-tests --doctest-extension=rst doc/
+	$(PYTESTS) --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/
 
 test-coverage: testing_data
 	rm -rf coverage .coverage
-	$(NOSETESTS) --with-coverage --cover-package=mne --cover-html --cover-html-dir=coverage
-
-test-profile: testing_data
-	$(NOSETESTS) --with-profile --profile-stats-file stats.pf mne
-	hotshot2dot stats.pf | dot -Tpng -o profile.png
+	$(PYTESTS) --cov=mne --cov-report html:coverage
+# whats the difference with test-no-sample-with-coverage?
 
 test-mem: in testing_data
-	ulimit -v 1097152 && $(NOSETESTS)
+	ulimit -v 1097152 && $(PYTESTS) mne
 
 trailing-spaces:
 	find . -name "*.py" | xargs perl -pi -e 's/[ \t]*$$//'
@@ -111,7 +108,7 @@ pydocstyle:
 
 docstring:
 	@echo "Running docstring tests"
-	@$(NOSETESTS) mne/tests/test_docstring_parameters.py
+	@$(PYTESTS) --doctest-modules mne/tests/test_docstring_parameters.py
 
 pep:
 	@$(MAKE) -k flake pydocstyle docstring codespell-error

appveyor.yml

@@ -1,9 +1,8 @@
 environment:
   global:
       PYTHON: "C:\\conda"
-      MINICONDA_VERSION: "latest"
-      CONDA_DEPENDENCIES: "setuptools numpy scipy matplotlib scikit-learn nose mayavi pandas h5py PIL pyside"
-      PIP_DEPENDENCIES: "git+git://github.com/nipy/PySurfer.git nibabel nitime nose-timer nose-faulthandler"
+      CONDA_DEPENDENCIES: "setuptools numpy scipy matplotlib scikit-learn mayavi pandas h5py PIL pyside pytest"
+      PIP_DEPENDENCIES: "git+git://github.com/nipy/PySurfer.git nibabel nitime nose-timer nose-faulthandler pytest-attrib"
   matrix:
       - PYTHON_VERSION: "2.7"
         PYTHON_ARCH: "64"
@@ -23,4 +22,4 @@ build: false  # Not a C# project, build stuff at the test step instead.
 
 test_script:
   # Run the project tests, but (sadly) exclude ones that take a long time
-  - "nosetests --verbosity=2 -a !slow_test --with-timer --timer-top-n=30 --timer-ok 5 --timer-warning 30"
+  - "py.test --doctest-modules -a \"not slow_test\" --showlocals --durations=20 mne"

mne/io/tests/test_pick.py

@@ -8,7 +8,8 @@
 
 from mne import (pick_channels_regexp, pick_types, Epochs,
                  read_forward_solution, rename_channels,
-                 pick_info, pick_channels, __file__, create_info)
+                 pick_info, pick_channels, create_info)
+from mne import __file__ as _root_init_fname
 from mne.io import (read_raw_fif, RawArray, read_raw_bti, read_raw_kit,
                     read_info)
 from mne.io.pick import (channel_indices_by_type, channel_type,
@@ -258,7 +259,7 @@ def test_picks_by_channels():
 def test_clean_info_bads():
     """Test cleaning info['bads'] when bad_channels are excluded."""
 
-    raw_file = op.join(op.dirname(__file__), 'io', 'tests', 'data',
+    raw_file = op.join(op.dirname(_root_init_fname), 'io', 'tests', 'data',
                        'test_raw.fif')
     raw = read_raw_fif(raw_file)
 

mne/stats/tests/test_cluster_level.py

@@ -406,7 +406,7 @@ def test_permutation_connectivity_equiv():
 
 
 @slow_test
-def spatio_temporal_cluster_test_connectivity():
+def test_spatio_temporal_cluster_connectivity():
     """Test spatio-temporal cluster permutations."""
     try:
         try: