split tests and run them in parallel

.github/workflows/pkgTests.yml

@@ -3,28 +3,58 @@ name: pynsee package tests
 on:
   push:
       branches:
-        - 'master'
+        - 'parallel_tests'
 
 jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        group: [1, 2, 3, 4, 5, 6, 7]
+        python-version: [3.10, 3.11, 3.12] # Python versions
+        exclude:        
+          - group: 1
+            python-version: 3.11
+          - group: 1
+            python-version: 3.12
+          - group: 2
+            python-version: 3.11
+          - group: 2
+            python-version: 3.12
+          - group: 3
+            python-version: 3.10
+          - group: 3
+            python-version: 3.12
+          - group: 4
+            python-version: 3.10
+          - group: 4
+            python-version: 3.12
+          - group: 5
+            python-version: 3.10
+          - group: 5
+            python-version: 3.11
+          - group: 6
+            python-version: 3.10
+          - group: 6
+            python-version: 3.11
+          - group: 7
+            python-version: 3.10
+          - group: 7
+            python-version: 3.11
 
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
+    - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        #sudo apt-get install libgeos-dev
         python -m pip install --upgrade pip
-        pip install flake8 pytest pytest-cov geopandas nbconvert matplotlib descartes parameterized 
+        pip install flake8 pytest pytest-cov pytest-xdist geopandas nbconvert matplotlib descartes parameterized 
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
         if [ -f requirements-extra.txt ]; then pip install -r requirements-extra.txt; fi
+        pip install .[full] 
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
@@ -36,9 +66,6 @@ jobs:
         sirene_key: ${{ secrets.SIRENE_KEY }}
       run: |
         pip install jupytext
-        pip install -r requirements.txt
-        pip install .[full]     
-        pip install -r requirements-extra.txt
         # test all notebooks        
         cd docs/examples 
         jupytext --to py *.md
@@ -48,13 +75,9 @@ jobs:
     - name: Test with pytest
       env:      
         sirene_key: ${{ secrets.SIRENE_KEY }}
+        TEST_GROUP: ${{ matrix.group }}
       run: |
-        pytest -v --cov
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v1
-      if: github.ref == 'refs/heads/master'
-      with:
-        token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
+          python split_tests.py --group $TEST_GROUP --total-groups 7
+          pytest -v --cov $(cat test_files.txt) -n auto
         
    

split_tests.py

@@ -0,0 +1,36 @@
+import os
+import argparse
+from pathlib import Path
+from glob import glob
+
+def get_test_files(test_dir="tests"):
+    """Retrieve all test files from the test directory."""
+    # return sorted(Path(test_dir).rglob("test_*.py"))
+    return sorted(glob("tests" + "/**/" + "test_*.py", recursive=True))
+
+def split_tests(test_files, group, total_groups):
+    """Split test files into groups."""
+    return [file for i, file in enumerate(test_files) if i % total_groups == group - 1]
+
+def main():
+    parser = argparse.ArgumentParser(description="Split test files for parallel execution.")
+    parser.add_argument("--group", type=int, required=True, help="Group number (1-indexed).")
+    parser.add_argument("--total-groups", type=int, required=True, help="Total number of groups.")
+    parser.add_argument("--test-dir", default="tests", help="Directory containing test files.")
+
+    args = parser.parse_args()
+    test_files = get_test_files(args.test_dir)
+    print(f"{test_files=}")
+    group_files = split_tests(test_files, args.group, args.total_groups)
+    print(f"{group_files=}")
+
+    if not group_files:
+        print(f"No tests to run for group {args.group}")
+    else:
+        # Write the list of test files to a text file for pytest to use
+        with open("test_files.txt", "w") as f:
+            for file in group_files:
+                f.write(f"{file}\n")
+
+if __name__ == "__main__":
+    main()

tests/download/test_pynsee_download.py

@@ -24,6 +24,8 @@ class MyTests(unittest.TestCase):
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
 
+    version=True
+
     if version:
 
         def test_check_url(self):

tests/geodata/test_pynsee_geodata.py

@@ -31,6 +31,7 @@ class TestFunction(TestCase):
 
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
+    version=True
 
     if version:
 

tests/localdata/test_pynsee_localdata.py

@@ -37,6 +37,8 @@ class TestFunction(TestCase):
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
 
+    version=True
+
     if version:
 
         def test_get_population(self):

tests/macrodata/test_pynsee_macrodata.py

@@ -47,6 +47,7 @@ class TestFunction(TestCase):
 
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
+    version=True
 
     if version:
 

tests/metadata/test_pynsee_metadata.py

@@ -23,6 +23,7 @@ class TestFunction(TestCase):
 
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
+    version=True
 
     if version:
 

tests/sirene/test_pynsee_sirene.py

@@ -25,6 +25,7 @@ class TestFunction(TestCase):
 
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
+    version=True
 
     if version:
 

tests/utils/test_pynsee_utils.py

@@ -24,6 +24,7 @@ class TestFunction(TestCase):
 
     test_onyxia = re.match(".*onyxia.*", os.getcwd())
     version = version or test_onyxia
+    version=True
 
     if version:
         StartKeys = _get_credentials()