Dynamically mark tests as slow/veryslow during pytest collection phase

.ci/generate_durations_log.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+ABS_REPO_PATH=$(unset CDPATH && cd "$(dirname "$0")/.." && echo $PWD)
+python3 -m pytest --durations 0 --slow --veryslow >$ABS_REPO_PATH/.ci/durations.log

.ci/parse_durations_log.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+
+import os
+import json
+
+ci_folder = os.path.dirname(__file__)
+
+def read_log():
+    stop_token = '= FAILURES ='
+    start_token = '= slowest test durations ='
+    start_token_seen = False
+    for line in open(os.path.join(ci_folder, 'durations.log')):
+        if stop_token in line:
+            return
+        elif start_token_seen:
+            time, kind, test_id = line.split()
+            if kind != 'call':
+                continue
+            if time[-1] != 's':
+                raise NotImplementedError("expected seconds")
+            yield test_id, float(time[:-1])
+        elif start_token in line:
+            start_token_seen = True
+
+
+def main(limits=(10, .1)):
+    """
+    parses durations.log (see generate_durations_log.sh for how to generate that file)
+    """
+    groupings = [[] for _ in range(len(limits))]
+    for test_id, time in read_log():
+        for idx, lim in enumerate(limits):
+            if time >= lim:
+                groupings[idx].append(test_id)
+    open(os.path.join(ci_folder, 'durations.json'), 'wt').write(json.dumps([sorted(gr) for gr in groupings]))
+
+
+if __name__ == '__main__':
+    main()

.gitignore

@@ -71,3 +71,6 @@ sample.tex
 
 # pytest cache folder
 .cache
+
+# pytest related data file for slow tests
+.ci/durations.log

conftest.py

@@ -2,21 +2,50 @@
 
 from __future__ import print_function, division, absolute_import
 
+import os
+import json
+
 import pytest
 
 
+
+durations_path = os.path.join('.ci', 'durations.json')
+if os.path.exists(durations_path):
+    veryslow_group, slow_group = json.loads(open(durations_path, 'rt').read())
+else:
+    veryslow_group, slow_group = None, None
+
+
 def pytest_addoption(parser):
     parser.addoption("--slow", dest="runslow", action="store_true",
                      help="allow slow tests to run")
+    parser.addoption("--veryslow", dest="runveryslow", action="store_true",
+                     help="allow very slow tests to run")
 
 
 def pytest_configure(config):
     # register an additional marker
     config.addinivalue_line("markers", "slow: slow test")
+    config.addinivalue_line("markers", "veryslow: very slow test")
 
 
 def pytest_runtest_setup(item):
     if not isinstance(item, pytest.Function):
         return
+
+    if veryslow_group is not None:
+        if item.nodeid in veryslow_group:
+            pytest.skip("very slow test: pass --veryslow to run")
+            return
+
+    if slow_group is not None:
+        if item.nodeid in slow_group:
+            pytest.skip("slow test: pass --slow to run")
+            return
+
     if not item.config.getvalue("runslow") and hasattr(item.obj, 'slow'):
         pytest.skip("slow test: pass --slow to run")
+        return
+    if not item.config.getvalue("runveryslow") and hasattr(item.obj, 'veryslow'):
+        pytest.skip("very slow test: pass --veryslow to run")
+        return