Add github workflow tool

.github/workflows/build-test-macos-clvm.yml

@@ -53,7 +53,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
-
+# Omitted checking out blocks and plots repo Chia-Network/test-cache
 
     - name: Link home directory
       run: |
@@ -70,7 +70,7 @@ jobs:
         brew install boost
         sh install.sh
 
-
+# Omitted installing Timelord
 
     - name: Install developer requirements
       run: |

.github/workflows/build-test-ubuntu-clvm.yml

@@ -60,7 +60,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
-
+# Omitted checking out blocks and plots repo Chia-Network/test-cache
 
     - name: Link home directory
       run: |
@@ -82,7 +82,7 @@ jobs:
       run: |
         sh install.sh
 
-
+# Omitted installing Timelord
 
     - name: Install developer requirements
       run: |

.github/workflows/upload-pypi-source.yml

@@ -55,7 +55,8 @@ jobs:
 
     - name: Lint source with mypy
       run: |
-        mypy src tests
+        mypy --exclude config.py tests
+        mypy src
 
     - name: Build source distribution
       run: |

tests/README.md

@@ -0,0 +1,53 @@
+
+# Test CI Job generation
+
+The CI jobs for these tests are managed by `build-workflows.py`
+
+If you add a test file, or move one to another directory of a test, please run `build-workflows.py`.
+Tests are recognized by the file glob `test_*.py`.
+Changing the contents of a file does not require running `build-workflows.py`
+
+We currently use github actions. Default runners have two vcpus.
+The workflows are located in [../.github/workflows/](https://github.com/Chia-Network/chia-blockchain/tree/main/.github/workflows)
+
+The inputs to `build-workflows.py` are the templates in `runner-templates`, the main config.py file in this directory,
+and the optional `config.py` files in some test subdirectories.
+Files in the template directory ending in `include.yml` are included in jobs based on the per-directory settings.
+
+The generated workflows are output to ../.github/workflows/
+
+Each subdirectory below the directories `root_test_dirs` in config.py becomes a job in the github workflow matrix.
+If your jobs run too long, simply move some tests into new subdirectories, and run `build-workflows.py`.
+A workflow built from a parent directory does not include the tests in its subdirectories.
+The subdirectory jobs do not include the tests from their parents.
+
+## config.py
+
+In the top tests directory, [config.py](https://github.com/Chia-Network/chia-blockchain/tree/main/tests/config.py)
+contains the application settings and the per-directory default settings.
+
+Each directory has an optional `config.py` file, which can override the per-directory default settings.
+
+Per directory settings defaults:
+
+```
+parallel = False
+checkout_blocks_and_plots = True
+install_timelord = True
+job_timeout = 30
+```
+
+### Parallel test execution: `parallel = True`
+
+If you are certain that all the tests in a directory can run in parallel, set `parallel = True` in config.py in that directory.
+
+### Optional job stages
+
+Set `checkout_blocks_and_plots` to `False` to omit checking out the [test-cache](https://github.com/Chia-Network/test-cache) repo.
+
+Set `install_timelord` to `False` to omit the step of installing a Time Lord for your directory's job.
+
+### Job Timeout
+
+Set `job_timeout` to the number of minutes you want the CI system to wait before it kills your job.
+Add two or three minutes to allow for job setup.

tests/blockchain/config.py

@@ -0,0 +1 @@
+job_timeout = 60

tests/build-workflows.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+
+# Run from the current directory.
+
+import argparse
+import config
+import logging
+import subprocess
+from pathlib import Path
+from typing import List
+
+
+def subdirs(root_dirs: List[str]) -> List[Path]:
+    dirs: List[Path] = []
+    for r in root_dirs:
+        dirs.extend(Path(r).rglob("**/"))
+    return [d for d in dirs if not (any(c.startswith("_") for c in d.parts) or any(c.startswith(".") for c in d.parts))]
+
+
+def module_dict(module):
+    return {k: v for k, v in module.__dict__.items() if not k.startswith("_")}
+
+
+def dir_config(dir):
+    import importlib
+
+    module_name = str(dir).replace("/", ".") + ".config"
+    try:
+        return module_dict(importlib.import_module(module_name))
+    except ModuleNotFoundError:
+        return {}
+
+
+def read_file(filename):
+    with open(filename) as f:
+        return f.read()
+    return None
+
+
+# input file
+def workflow_yaml_template_text(os):
+    return Path(f"runner-templates/build-test-{os}").read_text()
+
+
+# output file
+def workflow_yaml_file(os, test_name):
+    return Path(f"../.github/workflows/build-test-{os}-{test_name}.yml")
+
+
+# test dir to name
+def test_name(dir):
+    return str(dir).replace("/", "-")
+
+
+def transform_template(template_text, replacements):
+    t = template_text
+    for r, v in replacements.items():
+        t = t.replace(r, v)
+    return t
+
+
+def test_files_in_dir(dir):
+    g = dir.glob("test_*.py")
+    return [] if g is None else [f for f in g]
+
+
+# -----
+
+default_replacements = {
+    "INSTALL_TIMELORD": read_file("runner-templates/install-timelord.include.yml").rstrip(),
+    "CHECKOUT_TEST_BLOCKS_AND_PLOTS": read_file("runner-templates/checkout-test-plots.include.yml").rstrip(),
+    "TEST_DIR": "",
+    "TEST_NAME": "",
+    "PYTEST_PARALLEL_ARGS": "",
+}
+
+# -----
+
+
+# replace with update_config
+def generate_replacements(defaults, conf, dir, test_files):
+    assert len(test_files) > 0
+    replacements = dict(defaults)
+
+    if not conf["checkout_blocks_and_plots"]:
+        replacements[
+            "CHECKOUT_TEST_BLOCKS_AND_PLOTS"
+        ] = "# Omitted checking out blocks and plots repo Chia-Network/test-cache"
+    if not conf["install_timelord"]:
+        replacements["INSTALL_TIMELORD"] = "# Omitted installing Timelord"
+    if conf["parallel"]:
+        replacements["PYTEST_PARALLEL_ARGS"] = "-n auto"
+    if conf["job_timeout"]:
+        replacements["JOB_TIMEOUT"] = str(conf["job_timeout"])
+    test_paths = ["tests/" + str(f) for f in test_files]
+    replacements["TEST_DIR"] = " ".join(test_paths)
+    replacements["TEST_NAME"] = test_name(str(dir))
+    if "test_name" in conf:
+        replacements["TEST_NAME"] = conf["test_name"]
+    return replacements
+
+
+# overwrite with directory specific values
+def update_config(parent, child):
+    if child is None:
+        return parent
+    conf = child
+    for k, v in parent.items():
+        if k not in child:
+            conf[k] = v
+    return conf
+
+
+# main
+arg_parser = argparse.ArgumentParser(description="Build github workflows")
+args = arg_parser.parse_args()
+
+test_dirs = subdirs(config.root_test_dirs)  # type: ignore
+
+for os in config.oses:  # type: ignore
+    template_text = workflow_yaml_template_text(os)
+    for dir in test_dirs:
+        test_files = test_files_in_dir(dir)
+        if len(test_files) == 0:
+            logging.info(f"Skipping {dir}: no tests collected")
+            continue
+        conf = update_config(module_dict(config), dir_config(dir))
+        replacements = generate_replacements(default_replacements, conf, dir, test_files)
+        txt = transform_template(template_text, replacements)
+        logging.info(f"Writing {os}-{test_name(dir)}")
+        workflow_yaml_file(os, test_name(dir)).write_text(txt)
+
+out = subprocess.run(["git", "diff", "../.github/workflows"])
+print(out.stdout)

tests/clvm/config.py

@@ -0,0 +1,3 @@
+parallel = True
+checkout_blocks_and_plots = False
+install_timelord = False

tests/config.py

@@ -0,0 +1,9 @@
+# Github actions template config
+oses = ["ubuntu", "macos"]
+root_test_dirs = ["blockchain", "clvm", "core", "simulation", "wallet"]
+
+# Defaults are conservative.
+parallel = False
+checkout_blocks_and_plots = True
+install_timelord = True
+job_timeout = 30

tests/core/full_node/config.py

@@ -0,0 +1 @@
+job_timeout = 60

tests/core/full_node/dos/config.py

@@ -0,0 +1 @@
+job_timeout = 60

tests/core/full_node/full_sync/config.py

@@ -0,0 +1 @@
+job_timeout = 60

tests/runner-templates/build-test-macos

@@ -0,0 +1,83 @@
+name: MacOS TEST_NAME Tests
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+        - '**'
+  pull_request:
+    branches:
+      - '**'
+
+jobs:
+  build:
+    name: MacOS TEST_NAME Tests
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: JOB_TIMEOUT
+    strategy:
+      fail-fast: false
+      max-parallel: 4
+      matrix:
+        python-version: [3.8, 3.9]
+        os: [macOS-latest]
+
+    steps:
+    - name: Cancel previous runs on the same branch
+      if: ${{ github.ref != 'refs/heads/dev' }}
+      uses: styfle/[email protected]
+      with:
+        access_token: ${{ github.token }}
+
+    - name: Checkout Code
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - name: Setup Python environment
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - name: Get pip cache dir
+      id: pip-cache
+      run: |
+        echo "::set-output name=dir::$(pip cache dir)"
+
+    - name: Cache pip
+      uses: actions/[email protected]
+      with:
+        # Note that new runners may break this https://github.com/actions/cache/issues/292
+        path: ${{ steps.pip-cache.outputs.dir }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+
+CHECKOUT_TEST_BLOCKS_AND_PLOTS
+
+    - name: Link home directory
+      run: |
+        cd $HOME
+        ln -s $GITHUB_WORKSPACE/.chia
+        echo "$HOME/.chia"
+        ls -al $HOME/.chia
+
+    - name: Run install script
+      env:
+        INSTALL_PYTHON_VERSION: ${{ matrix.python-version }}
+        BUILD_VDF_CLIENT: "N"
+      run: |
+        brew install boost
+        sh install.sh
+
+INSTALL_TIMELORD
+
+    - name: Install developer requirements
+      run: |
+        . ./activate
+        venv/bin/python -m pip install pytest pytest-asyncio pytest-xdist
+
+    - name: Test TEST_NAME code with pytest
+      run: |
+        . ./activate
+        ./venv/bin/py.test TEST_DIR -s -v --durations 0 PYTEST_PARALLEL_ARGS