move tests into visible 'tests' folder. make test_all python script, …

…and also update target-generating script (which helps with generating tests). reduce github-specific part to simply installing relevant dependencies and running the the test_all script.

.github/workflows/test-python.yml

@@ -20,22 +20,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install antlr4-python3-runtime==4.9.1
-      - name: pipe in all the inputs and concatenate them
+      - name: run all the tests in the python script
         run: |
-          python3 .github/workflows/tests/runall.py
-      - name: make sure output is equivalent to target for individual files
-        run: |
-          diff .github/workflows/tests/out/smol.txt                       .github/workflows/tests/tgt/smol.txt 
-          diff .github/workflows/tests/out/simple_sequences_no_errors.txt .github/workflows/tests/tgt/simple_sequences_no_errors.txt 
-          diff .github/workflows/tests/out/some_errors.txt                .github/workflows/tests/tgt/some_errors.txt 
-          diff .github/workflows/tests/out/whitespace_okay.txt            .github/workflows/tests/tgt/whitespace_okay.txt 
-          diff .github/workflows/tests/out/paper_examples.txt             .github/workflows/tests/tgt/paper_examples.txt 
-          diff .github/workflows/tests/out/simple_assignments.txt         .github/workflows/tests/tgt/simple_assignments.txt 
-      - name: in case i ever forget to add a test to the yaml... make sure entirety of found files are equivalent when concatenated
-        run: |
-          diff concatenated_outs.txt concatenated_tgts.txt 
-      - name: move in broken rasplib to see if it prints errors for that properly
-        run: |
-          mv .github/workflows/tests/broken_rasplib/rasplib.rasp RASP_support/rasplib.rasp
-          python3 RASP_support/REPL.py <.github/workflows/tests/broken_rasplib/empty.txt >.github/workflows/tests/broken_rasplib/out.txt
-          diff .github/workflows/tests/broken_rasplib/out.txt .github/workflows/tests/broken_rasplib/tgt.txt
+          python3 tests/test_all.py

tests/make_tgts.py

@@ -0,0 +1,85 @@
+# python3 .github/workflows/tests/make_tgts.py
+# !!!!! FOR RUNNING ON 'CORRECT' REPL
+import os
+
+
+testpath = "tests"
+inpath = testpath+"/in"
+outpath = testpath+"/out"
+tgtpath = testpath+"/tgt"
+libtestspath = testpath+"/broken_libs"
+libspath = libtestspath+"/lib"
+libtgtspath = libtestspath+"/tgt"
+liboutspath = libtestspath+"/out"
+
+curr_path_marker = "[current]"
+
+
+REPL_path = "RASP_support/REPL.py"
+rasplib_path = "RASP_support/rasplib.rasp"
+
+def things_in_path(path):
+	if not os.path.exists(path):
+		return []
+	return [p for p in os.listdir(path) if not p==".DS_Store"]
+
+def joinpath(*a):
+	return "/".join(a)
+
+for p in [tgtpath,libtgtspath]:
+	if not os.path.exists(p):
+		os.makedirs(p)
+
+all_names = things_in_path(inpath)
+
+def fix_file_paths(filename,curr_path_marker):
+	mypath  = os.path.abspath(".")
+
+	with open(filename,"r") as f:
+		filecontents = "".join(f)
+
+	filecontents = filecontents.replace(mypath,curr_path_marker)
+
+	with open(filename,"w") as f:
+		print(filecontents,file=f)
+
+
+def run_input(name):
+	os.system("python3 "+REPL_path+" <"+inpath+"/"+name+" >"+tgtpath+"/"+name)
+	fix_file_paths(tgtpath+"/"+name,curr_path_marker)
+
+def run_inputs():
+	print("making the target outputs!")
+	for n in all_names:
+		run_input(n)
+
+
+def run_broken_lib(l):
+	os.system("cp "+joinpath(libspath,l)+" "+rasplib_path)
+	os.system("python3 "+REPL_path+" <"+joinpath(libtestspath,"empty.txt")+ " >"+joinpath(libtgtspath,l))
+
+
+real_rasplib_safe_place = "make_tgts_helper/temp"
+safe_rasplib_name = "safe_rasplib.rasp"
+
+def save_rasplib():
+	if not os.path.exists(real_rasplib_safe_place):
+		os.makedirs(real_rasplib_safe_place)
+	os.system("mv "+rasplib_path+" "+joinpath(real_rasplib_safe_place,safe_rasplib_name))
+
+def restore_rasplib():
+	os.system("mv "+joinpath(real_rasplib_safe_place,safe_rasplib_name)+" "+rasplib_path)
+
+
+def run_broken_libs():
+	print("making the broken lib targets!")
+	save_rasplib()
+	all_libs = things_in_path(libspath)
+	for l in all_libs:
+		run_broken_lib(l)
+	restore_rasplib()
+
+
+if __name__ == "__main__":
+	run_inputs()
+	run_broken_libs()

tests/run_all.py

@@ -0,0 +1,64 @@
+import os
+from make_tgts import fix_file_paths, curr_path_marker, joinpath, things_in_path, \
+testpath, inpath, outpath, tgtpath, libtestspath, libspath, libtgtspath, liboutspath, \
+save_rasplib, restore_rasplib
+
+
+def check_equal(f1,f2):
+	res = os.system("diff "+f1+" "+f2)
+	return res == 0 # 0 = diff found no differences
+
+
+for p in [outpath,liboutspath]:
+	if not os.path.exists(p):
+		os.makedirs(p)
+
+def run_input(name):
+	os.system("python3 RASP_support/REPL.py <"+joinpath(inpath,name)+" >"+joinpath(outpath,name))
+	fix_file_paths(joinpath(outpath,name),curr_path_marker)
+	return check_equal(joinpath(outpath,name),joinpath(tgtpath,name))
+
+def run_inputs():
+	all_names = things_in_path(inpath)
+	passed = True
+	for n in all_names:
+		success = run_input(n)
+		print("input",n,"passed:",success)
+		if not success:
+			passed = False
+	return passed
+
+def test_broken_lib(l):
+	os.system("cp "+joinpath(libspath,l)+" RASP_support/rasplib.rasp")
+	os.system("python3 RASP_support/REPL.py <"+joinpath(libtestspath,"empty.txt")+ " >"+joinpath(liboutspath,l))
+	return check_equal(joinpath(liboutspath,l),joinpath(libtgtspath,l))
+
+def run_broken_libs():
+	save_rasplib()
+	all_libs = things_in_path(libspath)
+	passed = True
+	for l in all_libs:
+		success = test_broken_lib(l)
+		print("lib",l,"passed (i.e., properly errored):",success)
+		if not success:
+			passed = False
+	restore_rasplib()
+	return passed
+
+
+
+if __name__ == "__main__":
+	passed_inputs = run_inputs()
+	print("passed all inputs:",passed_inputs)
+	print("=====\n\n=====")
+	passed_broken_intialisations = run_broken_libs()
+	print("properly reports broken initialisations:",passed_broken_intialisations)
+	print("=====\n\n=====")
+
+
+	passed_everything = False not in [passed_inputs,passed_broken_intialisations]
+	print("=====\npassed everything:",passed_everything)
+	if passed_everything:
+		exit(0)
+	else:
+		exit(1)

tests/test_all.py

@@ -0,0 +1,64 @@
+import os
+from make_tgts import fix_file_paths, curr_path_marker, joinpath, things_in_path, \
+testpath, inpath, outpath, tgtpath, libtestspath, libspath, libtgtspath, liboutspath, \
+save_rasplib, restore_rasplib
+
+
+def check_equal(f1,f2):
+	res = os.system("diff "+f1+" "+f2)
+	return res == 0 # 0 = diff found no differences
+
+
+for p in [outpath,liboutspath]:
+	if not os.path.exists(p):
+		os.makedirs(p)
+
+def run_input(name):
+	os.system("python3 RASP_support/REPL.py <"+joinpath(inpath,name)+" >"+joinpath(outpath,name))
+	fix_file_paths(joinpath(outpath,name),curr_path_marker)
+	return check_equal(joinpath(outpath,name),joinpath(tgtpath,name))
+
+def run_inputs():
+	all_names = things_in_path(inpath)
+	passed = True
+	for n in all_names:
+		success = run_input(n)
+		print("input",n,"passed:",success)
+		if not success:
+			passed = False
+	return passed
+
+def test_broken_lib(l):
+	os.system("cp "+joinpath(libspath,l)+" RASP_support/rasplib.rasp")
+	os.system("python3 RASP_support/REPL.py <"+joinpath(libtestspath,"empty.txt")+ " >"+joinpath(liboutspath,l))
+	return check_equal(joinpath(liboutspath,l),joinpath(libtgtspath,l))
+
+def run_broken_libs():
+	save_rasplib()
+	all_libs = things_in_path(libspath)
+	passed = True
+	for l in all_libs:
+		success = test_broken_lib(l)
+		print("lib",l,"passed (i.e., properly errored):",success)
+		if not success:
+			passed = False
+	restore_rasplib()
+	return passed
+
+
+
+if __name__ == "__main__":
+	passed_inputs = run_inputs()
+	print("passed all inputs:",passed_inputs)
+	print("=====\n\n=====")
+	passed_broken_intialisations = run_broken_libs()
+	print("properly reports broken initialisations:",passed_broken_intialisations)
+	print("=====\n\n=====")
+
+
+	passed_everything = False not in [passed_inputs,passed_broken_intialisations]
+	print("=====\npassed everything:",passed_everything)
+	if passed_everything:
+		exit(0)
+	else:
+		exit(1)

.github/workflows/tests/tgt/paper_examples.txt → tests/tgt/paper_examples.txt

@@ -23,3 +23,4 @@ running example is: hello
 >> 	 =  [P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, P, T, P, P, P, P, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F, F] (strings)
 >> 	 =  [P, T, P, T, P, T, P, P, P, P, P, P, P, P, P, T, P, P, P, T, P, P, P, T, P, T, P, T, P, F, F, F, F, F, F, F, F, F, F, F, F, F, F] (strings)
 >> .. .. .. 
+

.github/workflows/tests/tgt/simple_assignments.txt → tests/tgt/simple_assignments.txt

@@ -8,3 +8,4 @@ running example is: hello
 >> .. ..     value: e =  13
     value: f =  5
 >> 
+

.github/workflows/tests/tgt/simple_sequences_no_errors.txt → tests/tgt/simple_sequences_no_errors.txt

@@ -23,3 +23,4 @@ running example is: hello
 >>      s-op: out
  	 Example: out("hello there") = [h, a, l, a, o, a, t, a, e, a, e] (strings)
 >> .. 
+

.github/workflows/tests/tgt/smol.txt → tests/tgt/smol.txt

@@ -3,3 +3,4 @@ running example is: hello
 >>      s-op: tokens
  	 Example: tokens("hello") = [h, e, l, l, o] (strings)
 >> .. .. .. 
+

.github/workflows/tests/tgt/some_errors.txt → tests/tgt/some_errors.txt

@@ -25,19 +25,19 @@ plain unfinished tuple
 ([h, e, l, l, o] (strings),)
 ===============================================================
 ===============================================================
-  File "/home/runner/work/RASP/RASP/RASP_support/FunctionalSupport.py", line 369, in <lambda>
+  File "[current]/RASP_support/FunctionalSupport.py", line 369, in <lambda>
     parents2res = lambda s,vt:_aggregate(s,vt,elementwise_function,default=default)
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 110, in aggregate
+  File "[current]/RASP_support/Support.py", line 110, in aggregate
     return to_sequences(apply_average_select(select,k_vars,func,default))
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 189, in apply_average_select
+  File "[current]/RASP_support/Support.py", line 189, in apply_average_select
     means_per_index = [apply_and_average_single_index(candidates_by_varname,
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 189, in <listcomp>
+  File "[current]/RASP_support/Support.py", line 189, in <listcomp>
     means_per_index = [apply_and_average_single_index(candidates_by_varname,
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 185, in apply_and_average_single_index
+  File "[current]/RASP_support/Support.py", line 185, in apply_and_average_single_index
     return tuple(mean(index_scores,o_by_i) for o_by_i in outputs_by_varname) # return_sequences expects multiple outputs to be in tuple form
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 185, in <genexpr>
+  File "[current]/RASP_support/Support.py", line 185, in <genexpr>
     return tuple(mean(index_scores,o_by_i) for o_by_i in outputs_by_varname) # return_sequences expects multiple outputs to be in tuple form
-  File "/home/runner/work/RASP/RASP/RASP_support/Support.py", line 178, in mean
+  File "[current]/RASP_support/Support.py", line 178, in mean
     raise Exception("asked to average multiple values, but they are non-numbers: "+str(vals))
 
  	 Example: out("hello") = EVALUATION FAILURE
@@ -51,3 +51,4 @@ plain unfinished tuple
 >>      dict: out = {}
 >> 	!! antlr exception: mismatched input 'if' expecting {'set', 's-op', 'selector', 'show', 'full seq display', 'examples', 'exit()', 'exit', 'quit', 'quit()', 'load', 'draw', '(', 'def', '{', 'for', 'aggregate(', '[', 'not', '-', '+', 'round', 'indicator', 'select(', 'range(', 'zip(', 'len(', Float, PosInt, String, Comment, ID} 	-- ignoring input
 >> .. 
+

.github/workflows/tests/tgt/whitespace_okay.txt → tests/tgt/whitespace_okay.txt

@@ -13,3 +13,4 @@ running example is: hello
  			 l | 1 1 1    
  			 o | 1 1 1 1  
 >> 
+