ENH: Make Qt default browser, add scraper (mne-tools#9960)

Co-authored-by: Martin Schulz <[email protected]>

.circleci/config.yml

@@ -1,7 +1,11 @@
-# Tagging a commit with:
-# - [circle front] will build the front page and perform test-doc
-# - [circle full] will build everything
-# - [circle linkcheck] will run our linkcheck
+# By default, for PRs CircleCI will build only examples that have changed.
+# For main commits, builds are skipped entirely, as we only do full builds
+# scheduled for one time daily.
+#
+# Tagging a commit with the following overrides these behaviors:
+# - [circle front] will run the front page examples and perform test-doc
+# - [circle full] will run all examples and perform test-doc
+# - [circle linkcheck] will run our linkcheck job
 # - [circle deploy] on a main or maint/* commit will try to immediately build
 #   and deploy docs rather than waiting for the nightly build
 
@@ -106,6 +110,8 @@ jobs:
               echo "export MNE_3D_OPTION_ANTIALIAS=false" >> $BASH_ENV
               source tools/get_minimal_commands.sh
               echo "export MNE_3D_BACKEND=pyvistaqt" >> $BASH_ENV
+              echo "export MNE_BROWSER_BACKEND=matplotlib" >> $BASH_ENV
+              echo "export MNE_BROWSER_PRECOMPUTE=false" >> $BASH_ENV
               echo "export PATH=~/.local/bin/:$PATH" >> $BASH_ENV
               echo "source ~/python_env/bin/activate" >> $BASH_ENV
               mkdir -p ~/.local/bin

.gitignore

@@ -57,6 +57,7 @@ doc/auto_tutorials/
 doc/modules/generated/
 doc/sphinxext/cachedir
 tutorials/intro/report.h5
+tutorials/intro/report_partial.hdf5
 tutorials/misc/report.h5
 tutorials/io/fnirs.csv
 pip-log.txt

Makefile

@@ -73,7 +73,7 @@ test-no-sample-with-coverage: in testing_data
 	$(PYTESTS) --cov=mne --cov-report html:coverage
 
 test-doc: sample_data testing_data
-	$(PYTESTS) --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --fulltrace
+	$(PYTESTS) --doctest-modules --doctest-ignore-import-errors --doctest-glob='*.rst' ./doc/ --ignore=./doc/auto_examples --ignore=./doc/auto_tutorials --ignore=./doc/_build --ignore=./doc/conf.py --ignore=doc/sphinxext --fulltrace
 
 test-coverage: testing_data
 	rm -rf coverage .coverage

doc/changes/latest.inc

@@ -209,6 +209,8 @@ Bugs
 
 API changes
 ~~~~~~~~~~~
+- The default browser for :meth:`raw.plot() <mne.io.Raw.plot>`, :meth:`epochs.plot() <mne.Epochs.plot>`, and :meth:`ica.plot_sources() <mne.preprocessing.ICA.plot_sources>` has been changed to the ``'qt'`` backend on systems where `mne_qt_browser <https://github.com/mne-tools/mne-qt-browser>`__ is installed. To change back to matplotlib within a session, you can use :func:`mne.viz.set_browser_backend('matplotlib') <mne.viz.set_browser_backend>`. To set it permanently on your system, you can use :func:`mne.set_config('MNE_BROWSER_BACKEND', 'matplotlib') <mne.set_config>` (:gh:`9960` by `Martin Schulz`_ and `Eric Larson`_)
+
 - ``mne.Info.pick_channels`` has been deprecated. Use ``inst.pick_channels`` to pick channels from :class:`~mne.io.Raw`, :class:`~mne.Epochs`, and :class:`~mne.Evoked`. Use :func:`mne.pick_info` to pick channels from :class:`mne.Info` (:gh:`10039` by `Mathieu Scheltienne`_)
 
 - All :func:`data_path <mne.datasets.sample.data_path>` functions now return :class:`python:pathlib.Path` objects rather than strings. Support for string concatenation with plus (``+``) is thus deprecated and will be removed in 1.2, use the forward-slash ``/`` operator instead (:gh:`10348` by `Eric Larson`_)

doc/conf.py

@@ -13,6 +13,7 @@
 import time
 import warnings
 from datetime import datetime, timezone
+import faulthandler
 
 import numpy as np
 import matplotlib
@@ -21,12 +22,15 @@
 from numpydoc import docscrape
 
 import mne
+from mne.fixes import _compare_version
 from mne.tests.test_docstring_parameters import error_ignores
 from mne.utils import (linkcode_resolve, # noqa, analysis:ignore
                        _assert_no_instances, sizeof_fmt, run_subprocess)
 from mne.viz import Brain  # noqa
 
 matplotlib.use('agg')
+faulthandler.enable()
+os.environ['_MNE_BROWSER_NO_BLOCK'] = 'true'
 
 # -- Path setup --------------------------------------------------------------
 
@@ -356,6 +360,13 @@ def __call__(self, gallery_conf, fname):
     report_scraper = mne.report._ReportScraper()
     scrapers += (report_scraper,)
     del backend
+try:
+    import mne_qt_browser
+    _min_ver = _compare_version(mne_qt_browser.__version__, '>=', '0.2')
+    if mne.viz.get_browser_backend() == 'qt' and _min_ver:
+        scrapers += (mne.viz._scraper._PyQtGraphScraper(),)
+except ImportError:
+    pass
 
 compress_images = ('images', 'thumbnails')
 # let's make things easier on Windows users
@@ -800,6 +811,8 @@ def reset_warnings(gallery_conf, fname):
         'ignore', '.*"is not" with a literal.*', module='nilearn')
     warnings.filterwarnings(  # scikit-learn FastICA whiten=True deprecation
         'ignore', r'.*From version 1\.3 whiten.*')
+    warnings.filterwarnings(  # seaborn -> pandas
+        'ignore', '.*iteritems is deprecated and will be.*')
     for key in ('HasTraits', r'numpy\.testing', 'importlib', r'np\.loads',
                 'Using or importing the ABCs from',  # internal modules on 3.7
                 r"it will be an error for 'np\.bool_'",  # ndimage
@@ -818,6 +831,8 @@ def reset_warnings(gallery_conf, fname):
                 'to mean no clipping',
                 r'the `scipy\.ndimage.*` namespace is deprecated',  # Dipy
                 '`np.MachAr` is deprecated',  # Numba
+                'distutils Version classes are deprecated',  # pydata-sphinx-th
+                'The module matplotlib.tight_layout is deprecated',  # nilearn
                 ):
         warnings.filterwarnings(  # deal with other modules having bad imports
             'ignore', message=".*%s.*" % key, category=DeprecationWarning)
@@ -1146,5 +1161,6 @@ def setup(app):
     sphinx_logger = sphinx.util.logging.getLogger('mne')
     sphinx_logger.info(
         f'Building documentation for MNE {release} ({mne.__file__})')
+    sphinx_logger.info(f'Building with scrapers={scrapers}')
     app.connect('build-finished', make_redirects)
     app.connect('build-finished', make_version)

examples/inverse/compute_mne_inverse_raw_in_label.py

@@ -48,7 +48,7 @@
                         start, stop, pick_ori=None)
 
 # Save result in stc files
-stc.save('mne_%s_raw_inverse_%s' % (method, label_name))
+stc.save('mne_%s_raw_inverse_%s' % (method, label_name), overwrite=True)
 
 # %%
 # View activation time-series

examples/time_frequency/source_power_spectrum.py

@@ -49,7 +49,7 @@
                          pick_ori="normal", n_fft=n_fft, label=label,
                          dB=True)
 
-stc.save('psd_dSPM')
+stc.save('psd_dSPM', overwrite=True)
 
 # %%
 # View PSD of sources in label

examples/time_frequency/source_space_time_frequency.py

@@ -58,7 +58,7 @@
                                  use_fft=False, n_jobs=1)
 
 for b, stc in stcs.items():
-    stc.save('induced_power_%s' % b)
+    stc.save('induced_power_%s' % b, overwrite=True)
 
 # %%
 # plot mean power

mne/conftest.py

@@ -66,7 +66,8 @@ def pytest_configure(config):
         config.addinivalue_line('markers', marker)
 
     # Fixtures
-    for fixture in ('matplotlib_config', 'close_all', 'check_verbose'):
+    for fixture in ('matplotlib_config', 'close_all', 'check_verbose',
+                    'qt_config'):
         config.addinivalue_line('usefixtures', fixture)
 
     # Warnings
@@ -135,6 +136,8 @@ def pytest_configure(config):
     # Jupyter notebook stuff
     ignore:.*unclosed context <zmq\.asyncio\.*:ResourceWarning
     ignore:.*unclosed event loop <.*:ResourceWarning
+    # TODO: This is indicative of a problem
+    ignore:.*Matplotlib is currently using agg.*:
     """  # noqa: E501
     for warning_line in warning_lines.split('\n'):
         warning_line = warning_line.strip()
@@ -180,6 +183,12 @@ def verbose_debug():
         yield
 
 
+@pytest.fixture(scope='session')
+def qt_config():
+    """Configure the Qt backend for viz tests."""
+    os.environ['_MNE_BROWSER_NO_BLOCK'] = 'true'
+
+
 @pytest.fixture(scope='session')
 def matplotlib_config():
     """Configure matplotlib for viz tests."""
@@ -208,7 +217,7 @@ def matplotlib_config():
     orig = cbook.CallbackRegistry
 
     class CallbackRegistryReraise(orig):
-        def __init__(self, exception_handler=None):
+        def __init__(self, exception_handler=None, signals=None):
             super(CallbackRegistryReraise, self).__init__(exception_handler)
 
     cbook.CallbackRegistry = CallbackRegistryReraise
@@ -434,8 +443,12 @@ def pg_backend(request, garbage_collect):
     """Use for pyqtgraph-specific test-functions."""
     _check_pyqtgraph(request)
     with use_browser_backend('qt') as backend:
+        backend._close_all()
         yield backend
         backend._close_all()
+        # This shouldn't be necessary, but let's make sure nothing is stale
+        import mne_qt_browser
+        mne_qt_browser._browser_instances.clear()
 
 
 @pytest.fixture(params=[
@@ -448,8 +461,13 @@ def browser_backend(request, garbage_collect):
     if backend_name == 'qt':
         _check_pyqtgraph(request)
     with use_browser_backend(backend_name) as backend:
+        backend._close_all()
         yield backend
         backend._close_all()
+        if backend_name == 'qt':
+            # This shouldn't be necessary, but let's make sure nothing is stale
+            import mne_qt_browser
+            mne_qt_browser._browser_instances.clear()
 
 
 @pytest.fixture(params=["pyvistaqt"])

mne/epochs.py

@@ -1163,7 +1163,7 @@ def plot(self, picks=None, scalings=None, n_epochs=20, n_channels=20,
              order=None, show=True, block=False, decim='auto', noise_cov=None,
              butterfly=False, show_scrollbars=True, show_scalebars=True,
              epoch_colors=None, event_id=None, group_by='type',
-             precompute='auto', use_opengl=None):
+             precompute=None, use_opengl=None):
         return plot_epochs(self, picks=picks, scalings=scalings,
                            n_epochs=n_epochs, n_channels=n_channels,
                            title=title, events=events, event_color=event_color,

mne/io/base.py

@@ -1540,7 +1540,7 @@ def plot(self, events=None, duration=10.0, start=0.0, n_channels=20,
              show_first_samp=False, proj=True, group_by='type',
              butterfly=False, decim='auto', noise_cov=None, event_id=None,
              show_scrollbars=True, show_scalebars=True, time_format='float',
-             precompute='auto', use_opengl=None, verbose=None):
+             precompute=None, use_opengl=None, verbose=None):
         return plot_raw(self, events, duration, start, n_channels, bgcolor,
                         color, bad_color, event_color, scalings, remove_dc,
                         order, show_options, title, show, block, highpass,

mne/preprocessing/ica.py

@@ -1951,7 +1951,7 @@ def plot_properties(self, inst, picks=None, axes=None, dB=True,
     def plot_sources(self, inst, picks=None, start=None,
                      stop=None, title=None, show=True, block=False,
                      show_first_samp=False, show_scrollbars=True,
-                     time_format='float', precompute='auto',
+                     time_format='float', precompute=None,
                      use_opengl=None):
         return plot_ica_sources(self, inst=inst, picks=picks,
                                 start=start, stop=stop, title=title, show=show,

mne/preprocessing/tests/test_ica.py

@@ -430,7 +430,8 @@ def test_ica_reset(method):
 @pytest.mark.parametrize('n_components', (2, 0.6))
 @pytest.mark.parametrize('noise_cov', (False, True))
 @pytest.mark.parametrize('n_pca_components', [20])
-def test_ica_core(method, n_components, noise_cov, n_pca_components):
+def test_ica_core(method, n_components, noise_cov, n_pca_components,
+                  browser_backend):
     """Test ICA on raw and epochs."""
     _skip_check_picard(method)
     raw = read_raw_fif(raw_fname).crop(0, stop).load_data()
@@ -499,13 +500,11 @@ def test_ica_core(method, n_components, noise_cov, n_pca_components):
     print(raw_sources)
 
     # test for gh-6271 (scaling of ICA traces)
-    fig = raw_sources.plot()
-    assert len(fig.mne.ax_main.lines) in (4, 8)
-    for line in fig.mne.ax_main.lines:
+    fig = raw_sources.plot(clipping=None)
+    assert len(fig.mne.traces) in (2, 6)
+    for line in fig.mne.traces:
         y = line.get_ydata()
-        if len(y) > 2:  # actual data, not markers
-            assert np.ptp(y) < 15
-    plt.close('all')
+        assert np.ptp(y) < 15
 
     sources = raw_sources[:, :][0]
     assert (sources.shape[0] == ica.n_components_)

mne/report/report.py

@@ -44,7 +44,7 @@
                      _check_ch_locs, _import_h5io_funcs)
 from ..viz import (plot_events, plot_alignment, plot_cov, plot_projs_topomap,
                    plot_compare_evokeds, set_3d_view, get_3d_backend,
-                   Figure3D)
+                   Figure3D, use_browser_backend)
 from ..viz.misc import _plot_mri_contours, _get_bem_plotting_surfaces
 from ..viz.utils import _ndarray_to_fig, tight_layout
 from ..forward import read_forward_solution, Forward
@@ -1508,7 +1508,8 @@ def _render_ica_properties(self, *, ica, picks, inst, n_jobs, image_format,
 
     def _render_ica_artifact_sources(self, *, ica, inst, artifact_type,
                                      image_format, tags):
-        fig = ica.plot_sources(inst=inst, show=False)
+        with use_browser_backend('matplotlib'):
+            fig = ica.plot_sources(inst=inst, show=False)
         _constrain_fig_resolution(
             fig, max_width=MAX_IMG_WIDTH, max_res=MAX_IMG_RES
         )
@@ -2669,10 +2670,11 @@ def _render_raw_butterfly_segments(
             raw.set_annotations(None)
 
             # Create the figure once and re-use it for performance reasons
-            fig = raw.plot(
-                butterfly=True, show_scrollbars=False, start=t_starts[0],
-                duration=durations[0], scalings=scalings, show=False
-            )
+            with use_browser_backend('matplotlib'):
+                fig = raw.plot(
+                    butterfly=True, show_scrollbars=False, start=t_starts[0],
+                    duration=durations[0], scalings=scalings, show=False
+                )
             _constrain_fig_resolution(
                 fig, max_width=MAX_IMG_WIDTH, max_res=MAX_IMG_RES
             )

mne/tests/test_epochs.py

@@ -18,7 +18,6 @@
                            assert_allclose, assert_equal, assert_array_less)
 import numpy as np
 from numpy.fft import rfft, rfftfreq
-import matplotlib.pyplot as plt
 import scipy.signal
 
 import mne
@@ -2730,7 +2729,7 @@ def make_epochs(picks, proj):
     assert_allclose(new_proj[n_meg:, n_meg:], np.eye(n_eeg), atol=1e-12)
 
 
-def test_array_epochs(tmp_path):
+def test_array_epochs(tmp_path, browser_backend):
     """Test creating epochs from array."""
     tempdir = str(tmp_path)
 
@@ -2768,7 +2767,6 @@ def test_array_epochs(tmp_path):
 
     # plotting
     epochs[0].plot()
-    plt.close('all')
 
     # indexing
     assert_array_equal(np.unique(epochs['1'].events[:, 2]), np.array([1]))

mne/utils/config.py

@@ -73,6 +73,7 @@ def set_memmap_min_size(memmap_min_size):
     'MNE_BROWSE_RAW_SIZE',
     'MNE_BROWSER_BACKEND',
     'MNE_BROWSER_USE_OPENGL',
+    'MNE_BROWSER_PRECOMPUTE',
     'MNE_CACHE_DIR',
     'MNE_COREG_ADVANCED_RENDERING',
     'MNE_COREG_COPY_ANNOT',
@@ -515,7 +516,7 @@ def sys_info(fid=None, show_paths=False, *, dependencies='user'):
     """  # noqa: E501
     _validate_type(dependencies, str)
     _check_option('dependencies', dependencies, ('user', 'developer'))
-    ljust = 21 if dependencies == 'developer' else 16
+    ljust = 21 if dependencies == 'developer' else 18
     platform_str = platform.platform()
     if platform.system() == 'Darwin' and sys.version_info[:2] < (3, 8):
         # platform.platform() in Python < 3.8 doesn't call

mne/utils/docs.py

@@ -1581,11 +1581,14 @@ def _reflow_param_docstring(docstring, has_first_line=True, width=75):
     Whether to load all data (not just the visible portion) into RAM and
     apply preprocessing (e.g., projectors) to the full data array in a separate
     processor thread, instead of window-by-window during scrolling. The default
-    ``'auto'`` compares available RAM space to the expected size of the
-    precomputed data, and precomputes only if enough RAM is available. ``True``
-    and ``'auto'`` only work if using the Qt backend.
+    None uses the ``MNE_BROWSER_PRECOMPUTE`` variable, which defaults to
+    ``'auto'``. ``'auto'`` compares available RAM space to the expected size of
+    the precomputed data, and precomputes only if enough RAM is available.
+    This is only used with the Qt backend.
 
     .. versionadded:: 0.24
+    .. versionchanged:: 1.0
+       Support for the MNE_BROWSER_PRECOMPUTE config variable.
 """
 
 docdict['use_opengl'] = """

mne/utils/tests/test_config.py

@@ -86,9 +86,9 @@ def test_sys_info():
     assert ('numpy:' in out)
 
     if platform.system() == 'Darwin':
-        assert 'Platform:       macOS-' in out
+        assert 'Platform:         macOS-' in out
     elif platform.system() == 'Linux':
-        assert 'Platform:       Linux' in out
+        assert 'Platform:         Linux' in out
 
 
 def test_get_subjects_dir(tmp_path, monkeypatch):

mne/viz/__init__.py

@@ -31,7 +31,7 @@
                                 set_3d_view, set_3d_title, create_3d_figure,
                                 close_3d_figure, close_all_3d_figures,
                                 get_brain_class)
-from . import backends
+from . import backends, _scraper
 from ._brain import Brain
 from ._figure import (get_browser_backend, set_browser_backend,
                       use_browser_backend)