gitignore and test cleanup

.gitignore

@@ -1,69 +1,39 @@
-# Scanpy
-.pytest_cache/
-.scanpy
-.coverage
-figures/
-cache/
-coverage.xml
-test.h5ad
-test_compr.h5ad
-test_no_compr.h5ad
-docs/_build
-docs/api/scanpy.api.*.rst
-docs/api/scanpy.*.rst
-!docs/api/scanpy.api.rst
-!docs/api/scanpy.api.AnnData.rst
-!docs/api/scanpy.plotting.rst
-archive/
-data/
-figs/
-*/figs*/
-write/
-/examples/
-scanpy.egg-info/
-scanpy_user/
-scanpy_save/
-scanpy_later/
-scripts_test/
-thirdp/
-test.py
-README.html
-scripts/
-test_notebooks.txt
-upload_to_pypi.sh
+# Scanpy outfiles
+/cache/
+/data/
+/write/
 
-# private files
-scanpy/examples/builtin_private.py
-scanpy/tools/paths.py
-scanpy/tools/tgdyn.py
-scanpy/tools/tgdyn_simple.py
-scripts/paths.py
-scripts/test_private.sh
-scripts/tgdyn.py
-scripts/tgdyn_simple.py
-.spyproject/
-scanpy/.spyproject/
+# Docs
+/docs/api/scanpy.*.rst
+!/docs/api/scanpy.api.rst
+!/docs/api/scanpy.api.AnnData.rst
+!/docs/api/scanpy.plotting.rstk
+
+# tests
+/.cache/
+/.pytest_cache/
+/scanpy/tests/test*.h5ad
+/scanpy/tests/figures/
+/scanpy/tests/notebooks/figures/
 
 # Environment management
-Pipfile
-Pipfile.lock
+/Pipfile
+/Pipfile.lock
 
 # always-ignore extensions
 *~
 
-# Python / Byte-compiled / optimized / DLL
+# Python build files
 __pycache__/
-*.py[cod]
-*.so
-.cache
+/dist/
+/build/
+/scanpy.egg-info/
 
-# OS or Editor files and folders
+# OS stuff
 .DS_Store
 Thumbs.db
 .ipynb_checkpoints/
 .directory
-/.idea/
 
-# always-ignore directories
-/dist/
-/build/
+# IDEs and editors
+/.idea/

scanpy/tests/conftest.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+import pytest
+
+import matplotlib as mpl
+mpl.use('agg')
+from matplotlib import pyplot
+from matplotlib.testing.compare import compare_images
+
+import scanpy
+
+scanpy.settings.verbosity = "hint"
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--internet-tests",
+        action="store_true",
+        default=False,
+        help="Run tests that retrieve stuff from the internet. This increases test time.",
+    )
+
+
+def pytest_collection_modifyitems(config, items):
+    run_internet = config.getoption("--internet-tests")
+    skip_internet = pytest.mark.skip(reason="need --internet-tests option to run")
+    for item in items:
+        # All tests marked with `pytest.mark.internet` get skipped unless
+        # `--run-internet` passed
+        if not run_internet and ("internet" in item.keywords):
+            item.add_marker(skip_internet)
+
+
+def make_comparer(path_expected: Path, path_actual: Path, *, tol: int):
+    def save_and_compare(basename, tolerance=None):
+        path_actual.mkdir(parents=True, exist_ok=True)
+        out_path = path_actual / f'{basename}.png'
+        pyplot.savefig(out_path, dpi=40)
+        pyplot.close()
+        if tolerance is None:
+            tolerance = tol
+        res = compare_images(str(path_expected / f'{basename}.png'), str(out_path), tolerance)
+        assert res is None, res
+    return save_and_compare
+
+
+@pytest.fixture
+def image_comparer():
+    return make_comparer
+
+
+@pytest.fixture
+def plt():
+    return pyplot

scanpy/tests/notebooks/test_paga_paul15_subsampled.py

@@ -3,32 +3,22 @@
 #
 # This is the subsampled notebook for testing.
 
-from matplotlib.testing import setup
-setup()
+from pathlib import Path
 
-import matplotlib as mpl
-mpl.use('agg')
-from matplotlib.testing.compare import compare_images
-import matplotlib.pyplot as pl
 import numpy as np
-import os
-import scanpy.api as sc
+from matplotlib.testing import setup
+setup()
 
-ROOT = os.path.dirname(os.path.abspath(__file__)) + '/_images_paga_paul15_subsampled/'
+import scanpy as sc
 
 
-def save_and_compare_images(basename):
-    if not os.path.exists('./figures/'): os.makedirs('./figures/')
-    outname = './figures/' + basename + '.png'
-    pl.savefig(outname, dpi=40)
-    pl.close()
-    tolerance = 25  # had to increase this for the heatmap
-                    # is not ideal and borderline high
-    res = compare_images(ROOT + '/' + basename + '.png', outname, tolerance)
-    assert res is None, res
+HERE: Path = Path(__file__).parent
+ROOT = HERE / '_images_paga_paul15_subsampled'
+FIGS = HERE / 'figures'
 
 
-def test_paga_paul15_subsampled():
+def test_paga_paul15_subsampled(image_comparer, plt):
+    save_and_compare_images = image_comparer(ROOT, FIGS, tol=25)
 
     adata = sc.datasets.paul15()
     sc.pp.subsample(adata, n_obs=200)
@@ -90,8 +80,8 @@ def test_paga_paul15_subsampled():
 
     adata.obs['distance'] = adata.obs['dpt_pseudotime']
 
-    _, axs = pl.subplots(ncols=3, figsize=(6, 2.5), gridspec_kw={'wspace': 0.05, 'left': 0.12})
-    pl.subplots_adjust(left=0.05, right=0.98, top=0.82, bottom=0.2)
+    _, axs = plt.subplots(ncols=3, figsize=(6, 2.5), gridspec_kw={'wspace': 0.05, 'left': 0.12})
+    plt.subplots_adjust(left=0.05, right=0.98, top=0.82, bottom=0.2)
     for ipath, (descr, path) in enumerate(paths):
         _, data = sc.pl.paga_path(
             adata, path, gene_names,

scanpy/tests/notebooks/test_pbmc3k.py

@@ -2,33 +2,34 @@
 # *First compiled on May 5, 2017. Updated August 14, 2018.*
 # # Clustering 3k PBMCs following a Seurat Tutorial
 #
-# This started out with a demonstration that Scanpy would allow to reproduce most of Seurat's ([Satija *et al.*, 2015](https://doi.org/10.1038/nbt.3192)) clustering tutorial as described on http://satijalab.org/seurat/pbmc3k_tutorial.html (July 26, 2017), which we gratefully acknowledge. In the meanwhile, we have added and removed several pieces.
+# This started out with a demonstration that Scanpy would allow to reproduce most of Seurat's
+# ([Satija *et al.*, 2015](https://doi.org/10.1038/nbt.3192)) clustering tutorial as described on
+# http://satijalab.org/seurat/pbmc3k_tutorial.html (July 26, 2017), which we gratefully acknowledge.
+# In the meanwhile, we have added and removed several pieces.
 #
-# The data consists in *3k PBMCs from a Healthy Donor* and is freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)).
+# The data consists in *3k PBMCs from a Healthy Donor* and is freely available from 10x Genomics
+# ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz)
+# from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)).
 
+from pathlib import Path
+
+import numpy as np
+import pytest
 
 from matplotlib.testing import setup
 setup()
-from matplotlib.testing.compare import compare_images
-import matplotlib.pyplot as pl
-import numpy as np
-import os
-import scanpy.api as sc
 
-ROOT = os.path.dirname(os.path.abspath(__file__)) + '/pbmc3k_images/'
+import scanpy as sc
 
 
-def save_and_compare_images(basename):
-    if not os.path.exists('./figures/'): os.makedirs('./figures/')
-    outname = './figures/' + basename + '.png'
-    pl.savefig(outname, dpi=40)
-    pl.close()
-    tolerance = 20
-    res = compare_images(ROOT + '/' + basename + '.png', outname, tolerance)
-    assert res is None, res
+HERE: Path = Path(__file__).parent
+ROOT = HERE / 'pbmc3k_images'
+FIGS = HERE / 'figures'
 
 
-def test_pbmc3k():
+@pytest.mark.internet
+def test_pbmc3k(image_comparer):
+    save_and_compare_images = image_comparer(ROOT, FIGS, tol=20)
 
     adata = sc.read('./data/pbmc3k_raw.h5ad', backup_url='http://falexwolf.de/data/pbmc3k_raw.h5ad')
 
@@ -43,13 +44,17 @@ def test_pbmc3k():
     mito_genes = [name for name in adata.var_names if name.startswith('MT-')]
     # for each cell compute fraction of counts in mito genes vs. all genes
     # the `.A1` is only necessary as X is sparse to transform to a dense array after summing
-    adata.obs['percent_mito'] = np.sum(
-        adata[:, mito_genes].X, axis=1).A1 / np.sum(adata.X, axis=1).A1
+    adata.obs['percent_mito'] = (
+        np.sum(adata[:, mito_genes].X, axis=1).A1 /
+        np.sum(adata.X, axis=1).A1
+    )
     # add the total counts per cell as observations-annotation to adata
     adata.obs['n_counts'] = adata.X.sum(axis=1).A1
 
-    sc.pl.violin(adata, ['n_genes', 'n_counts', 'percent_mito'],
-                 jitter=False, multi_panel=True, show=False)
+    sc.pl.violin(
+        adata, ['n_genes', 'n_counts', 'percent_mito'],
+        jitter=False, multi_panel=True, show=False,
+    )
     save_and_compare_images('violin')
 
     sc.pl.scatter(adata, x='n_counts', y='percent_mito', show=False)
@@ -65,7 +70,8 @@ def test_pbmc3k():
     sc.pp.normalize_per_cell(adata, counts_per_cell_after=1e4)
 
     filter_result = sc.pp.filter_genes_dispersion(
-        adata.X, min_mean=0.0125, max_mean=3, min_disp=0.5)
+        adata.X, min_mean=0.0125, max_mean=3, min_disp=0.5,
+    )
     sc.pl.filter_genes_dispersion(filter_result, show=False)
     save_and_compare_images('filter_genes_dispersion')
 
@@ -121,7 +127,8 @@ def test_pbmc3k():
         'CD4 T cells', 'CD14+ Monocytes',
         'B cells', 'CD8 T cells',
         'NK cells', 'FCGR3A+ Monocytes',
-        'Dendritic cells', 'Megakaryocytes']
+        'Dendritic cells', 'Megakaryocytes',
+    ]
     adata.rename_categories('louvain', new_cluster_names)
 
     # sc.pl.umap(adata, color='louvain', legend_loc='on data', title='', frameon=False, show=False)