Adding --cluster-sidecar for Slurm

.gitignore

@@ -1,2 +1,9 @@
 __pycache__
 .pytest
+
+# pycharm
+.idea/
+
+# text editors
+*.sw?
+*~

README.md

@@ -4,12 +4,14 @@
 
 This cookiecutter provides a template Snakemake profile for
 configuring Snakemake to run on the [SLURM Workload
-Manager](https://slurm.schedmd.com/). The profile defines three
-scripts
+Manager](https://slurm.schedmd.com/). The profile defines the
+following scripts
 
 1. `slurm-submit.py` - submits a jobscript to slurm
 2. `slurm-jobscript.sh` - a template jobscript
 3. `slurm-status.py` - checks the status of jobs in slurm
+4. `slurm-sidecar.py` - run a Snakemake cluster sidecar for caching
+   queries to Slurm's controller/database daemons
 
 and a configuration file `config.yaml` that defines default values for
 snakemake command line arguments. The default `config.yaml` file is
@@ -18,6 +20,7 @@ snakemake command line arguments. The default `config.yaml` file is
     jobscript: "slurm-jobscript.sh"
     cluster: "slurm-submit.py"
     cluster-status: "slurm-status.py"
+    cluster-sidecar: "slurm-sidecar.py"
     max-jobs-per-second: 1
     max-status-checks-per-second: 10
     local-cores: 1
@@ -27,7 +30,8 @@ Given an installed profile `profile_name`, when snakemake is run with
 `--profile profile_name`, the configuration above would imply the
 following snakemake call:
 
-    snakemake --jobscript slurm-jobscript.sh --cluster slurm-submit.py --cluster-status slurm-status.py --restart-times 3 --max-jobs-per-second 1 --max-status-checks-per-second 10 --local-cores 1 --latency-wait 60
+    snakemake --jobscript slurm-jobscript.sh --cluster slurm-submit.py --cluster-status slurm-status.py \
+        --restart-times 3 --max-jobs-per-second 1 --max-status-checks-per-second 10 --local-cores 1 --latency-wait 60
 
 plus any additional options to snakemake that the user has applied.
 

tests/mock-slurm/bin/sacct

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+cat <"EOF"
+1044785|RUNNING|0:0
+1044785.extern|RUNNING|0:0
+1044785.0|RUNNING|0:0
+EOF

tests/mock-slurm/bin/squeue

@@ -0,0 +1,7 @@
+#!/usr/bin/bash
+
+cat <<"EOF"
+JOBID,STATE
+1044785,RUNNING
+1044875,RUNNING
+EOF

tests/test_sidecar.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+import json
+import os
+import signal
+import subprocess
+import tempfile
+import time
+
+import pytest
+
+
+@pytest.mark.slow
+@pytest.mark.timeout(60)
+def test_cluster_sidecar_smoke():
+    env = dict(os.environ)
+    env["PATH"] = (
+        os.path.realpath(os.path.dirname(__file__) + "/mock-slurm/bin") + ":" + env.get("PATH")
+    )
+    path_sidecar_py = os.path.realpath(
+        os.path.dirname(__file__) + "/../{{cookiecutter.profile_name}}/slurm-sidecar.py"
+    )
+    with tempfile.TemporaryFile("w+t") as tmpf:
+        with subprocess.Popen(["python", path_sidecar_py], env=env, text=True, stdout=tmpf) as proc:
+            time.sleep(2)
+            os.kill(proc.pid, signal.SIGTERM)
+        tmpf.seek(0)
+        stdout = tmpf.read()
+    the_vars = json.loads(stdout.splitlines()[0])
+    assert "server_port" in the_vars
+    assert "server_secret" in the_vars
+    assert proc.returncode == 0

{{cookiecutter.profile_name}}/config.yaml

@@ -2,6 +2,8 @@ restart-times: 3
 jobscript: "slurm-jobscript.sh"
 cluster: "slurm-submit.py"
 cluster-status: "slurm-status.py"
+cluster-sidecar: "slurm-sidecar.py"
+cluster-cancel: "scancel"
 max-jobs-per-second: 1
 max-status-checks-per-second: 10
 local-cores: 1