FUNCZ-317 FUNCZ-349 Replace XML recipe with JSON/YAML.

Replaces the XML recipe implementation with the SONATA one, and also switches to SONATA circuit configurations.

.ci/generate_circuit_tests.sh

@@ -4,29 +4,23 @@ for i in $(seq 1 2); do
     for m in $(seq 0 $((${#modes[*]} - 1))); do
         fn="test_circuit${i}k_${modes[$m]}.sh"
         cat >$fn <<EOS
-part=\$([ \$(date +%H) -gt 8 -a \$(date +%H) -lt 20 ] && echo interactive || echo prod)
-
 export BASE=\$DATADIR/cellular/circuit-${i}k/
-export CIRCUIT=\$BASE/nodes.h5
-export MORPHOS=\$BASE/morphologies/h5
-export RECIPE=\$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=\$BASE/circuit_config.json
+export RECIPE=\$BASE/bioname/recipe.json
 export TOUCHES=\$BASE/touches/parquet/*.parquet
 
-salloc -Aproj16 -p\$part -Cnvme -N1 --exclusive --mem=0 \\
-    srun dplace functionlizer \\
-        -H \\
-        --${modes[$m]} \\
-        --output-dir=\$PWD \\
-        --checkpoint-dir=\$PWD \\
-        -p spark.master=spark://\\\$\\(hostname\\):7077 \\
-        --from \$CIRCUIT All --to \$CIRCUIT All \\
-        \$RECIPE \$MORPHOS \\
-        --touches \$TOUCHES
+srun dplace functionalizer \\
+    -H \\
+    --${modes[$m]} \\
+    --output-dir="\$PWD" \\
+    --checkpoint-dir="\$PWD" \\
+    --circuit-config=\$CIRCUIT \\
+    --recipe=\$RECIPE \\
+    -- \$TOUCHES
 
-parquet-coalesce circuit.parquet single.parquet
 parquet-compare \\
-    \$CIRCUIT \\
-    single.parquet \\
+    \${CIRCUIT%circuit_config.json}nodes.h5 \\
+    circuit.parquet \\
     \$BASE/touches/${labels[$m]}/circuit.parquet
 EOS
     done

.ci/test_circuit1k_gj.sh

@@ -1,15 +1,13 @@
 export BASE=$DATADIR/cellular/circuit-1k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
     -H \
     --gap-junctions \
     --output-dir="$PWD" \
     --checkpoint-dir="$PWD" \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES

.ci/test_circuit1k_s2f.sh

@@ -1,20 +1,18 @@
 export BASE=$DATADIR/cellular/circuit-1k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.reposition.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
     -H \
     --s2f \
     --output-dir="$PWD" \
     --checkpoint-dir="$PWD" \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES
 
 parquet-compare \
-    $CIRCUIT \
+    ${CIRCUIT%circuit_config.json}nodes.h5 \
     circuit.parquet \
     $BASE/touches/functional/circuit.parquet

.ci/test_circuit1k_s2f_split_and_merge.sh

@@ -1,8 +1,8 @@
 export BASE=$DATADIR/cellular/circuit-1k/
 export NODES=$BASE/nodes.h5
+export CIRCUIT=$BASE/circuit_config.json
 export NODESETS=$BASE/nodesets.json
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 for half in empty full; do
@@ -11,10 +11,9 @@ for half in empty full; do
         --s2f \
         --output-dir="$PWD/half_${half}_out" \
         --checkpoint-dir="$PWD/half_${half}_check" \
-        --from-nodeset $NODESETS half_$half \
-        --from $NODES All --to $NODES All \
-        --recipe $RECIPE \
-        --morphologies $MORPHOS \
+        --from-nodeset half_$half \
+        --circuit-config=$CIRCUIT \
+        --recipe=$RECIPE \
         -- $TOUCHES
 done
 

.ci/test_circuit1k_s2s.sh

@@ -1,20 +1,18 @@
 export BASE=$DATADIR/cellular/circuit-1k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
     -H \
     --s2s \
-    --output-dir=$PWD \
-    --checkpoint-dir=$PWD \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --output-dir="$PWD" \
+    --checkpoint-dir="$PWD" \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES
 
 parquet-compare \
-    $CIRCUIT \
+    ${CIRCUIT%circuit_config.json}nodes.h5 \
     circuit.parquet \
     $BASE/touches/structural/circuit.parquet

.ci/test_circuit1k_s2s_nodesets.sh

@@ -1,23 +1,21 @@
 export BASE=$DATADIR/cellular/circuit-1k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
     -H \
     --s2s \
     --output-dir=$PWD \
     --checkpoint-dir=$PWD \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --from-nodeset $BASE/nodesets.json test \
-    --to-nodeset $BASE/nodesets.json test \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --from-nodeset test \
+    --to-nodeset test \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES
 
 parquet-compare-ns \
-    $CIRCUIT \
+    ${CIRCUIT%circuit_config.json}nodes.h5 \
     $BASE/touches/structural/circuit.parquet \
     circuit.parquet \
     4

.ci/test_circuit2k_s2f.sh

@@ -1,18 +1,18 @@
 export BASE=$DATADIR/cellular/circuit-2k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
+    -H \
     --s2f \
-    --output-dir=$PWD \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --output-dir="$PWD" \
+    --checkpoint-dir="$PWD" \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES
 
 parquet-compare \
-    $CIRCUIT \
+    ${CIRCUIT%circuit_config.json}nodes.h5 \
     circuit.parquet \
     $BASE/touches/functional/circuit.parquet

.ci/test_circuit2k_s2s.sh

@@ -1,20 +1,18 @@
 export BASE=$DATADIR/cellular/circuit-2k/
-export CIRCUIT=$BASE/nodes.h5
-export MORPHOS=$BASE/morphologies/h5
-export RECIPE=$BASE/bioname/builderRecipeAllPathways.xml
+export CIRCUIT=$BASE/circuit_config.json
+export RECIPE=$BASE/bioname/recipe.json
 export TOUCHES=$BASE/touches/parquet/*.parquet
 
 srun dplace functionalizer \
     -H \
     --s2s \
-    --output-dir=$PWD \
-    --checkpoint-dir=$PWD \
-    --from $CIRCUIT All --to $CIRCUIT All \
-    --recipe $RECIPE \
-    --morphologies $MORPHOS \
+    --output-dir="$PWD" \
+    --checkpoint-dir="$PWD" \
+    --circuit-config=$CIRCUIT \
+    --recipe=$RECIPE \
     -- $TOUCHES
 
 parquet-compare \
-    $CIRCUIT \
+    ${CIRCUIT%circuit_config.json}nodes.h5 \
     circuit.parquet \
     $BASE/touches/structural/circuit.parquet

pyproject.toml

@@ -22,7 +22,7 @@ classifiers = [
 ]
 readme = "README.rst"
 dependencies = [
-    "fz-td-recipe",
+    "fz-td-recipe>=0.1.3.dev0",
     "h5py",
     "hdfs",
     "jprops",
@@ -32,7 +32,7 @@ dependencies = [
     "numpy",
     "packaging",
     "pandas",
-    "pyarrow<11",
+    "pyarrow<15",
     "pyspark>=3",
 ]
 
@@ -56,6 +56,9 @@ metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
 
 [tool.setuptools_scm]  # Section required
 
+[tool.ruff]
+line-length = 100
+
 [tool.black]
 extend-exclude = 'deps\/.*$'
 line_length = 100

src/sparkmanager/manager.py

@@ -56,7 +56,7 @@ def finish():
             """Save the final runtime upon object deletion."""
             now = time.time()
             self.__report["runtime"].append((now - self.__start, self.__start, now))
-            if os.path.exists(self.__eventlog):
+            if self.__eventlog and os.path.exists(self.__eventlog):
                 try:
                     log = EventLog(self.__eventlog)
                     self.__report["spark"]["shuffle"] = log.shuffle_size

src/spykfunc/__init__.py

@@ -17,10 +17,11 @@
 ################################################################################
 
 """Spykfunc - An implementation of Functionalizer in PySpark."""
+
 import logging as _log
-import pkg_resources as _pkg
+import importlib.metadata as _meta
 
-__version__ = _pkg.get_distribution(__name__).version
+__version__ = _meta.version(__name__)
 
 
 # General logging