Upped testcode to support possibly failing tests.

Also, improved the management of verbosity in tests.

test-suite2/run_tests

@@ -121,13 +121,18 @@ if __name__ == "__main__":
             print >> sys.stderr, "You cannot specify both --default and --category"
             sys.exit(1)
 
+
+    verbose_cmd = ["--verbose"] 
     new_env = os.environ.copy()
     if args.verbose:
         new_env["W90VERBOSETESTS"] = "true"
+        # Put twice --verbose, to set the verbosity level at 3
+        # and therefore, e.g. print the actual command being run
+        # (with mpirun etc.)
+        verbose_cmd *= 2
 
     command_pieces = [
-        testcode_exe,
-        "--verbose", 
+        testcode_exe] + verbose_cmd + [
         "--category={}".format(the_category),
         "--jobconfig={}".format(jobconfig),
         "--userconfig={}".format(userconfig)
@@ -136,8 +141,10 @@ if __name__ == "__main__":
     if args.numprocs:
         command_pieces.append("--processors={}".format(args.numprocs))
 
-    print "I will run the following command:"
-    print " ".join(command_pieces)   
+    if args.verbose:
+        print "I will run the following command:"
+        print " ".join(command_pieces)   
+
     return_code = subprocess.call(
         command_pieces
         , stdout=sys.stdout, stderr=sys.stderr, env=new_env)

test-suite2/testcode-README.txt

@@ -5,3 +5,7 @@ stripped out of the documentation.
 This has been retrieved on March 29, 2017
 (master branch, from commit 2258047a0d09714898572244f274f8dc47efadf3).
 
+This has been then modified according to the PR https://github.com/jsspencer/testcode/pull/10
+(commit 75bd8204a565e0023f515e50fb110b6629d8092b]
+
+

test-suite2/testcode/lib/testcode2/__init__.py

@@ -82,6 +82,9 @@ def __init__(self, name, exe, test_id, benchmark, **kwargs):
         self.skip_args = ''
         self.verify = False
         self.extract_fn = None
+        # By default, the job is expected to exit with error code 0.
+        # Setting it to True will discard the exit status/error code.
+        self.can_fail = False
 
         # Info
         self.vcs = None
@@ -312,10 +315,11 @@ def run_test(self, verbose=1, cluster_queue=None, rundir=None):
                             err.append(sys.exc_info()[1])
                     status = validation.Status()
                     if job.returncode != 0:
-                        err.insert(0, 'Error running job.  Return code: %i'
-                                        % job.returncode)
-                        (status, msg) = self.skip_job(test_input, test_arg,
-                                                      verbose)
+                        if not self.test_program.can_fail:
+                            err.insert(0, 'Error running job.  Return code: %i'
+                                            % job.returncode)
+                            (status, msg) = self.skip_job(test_input, test_arg,
+                                                          verbose)
                     if status.skipped():
                         self._update_status(status, (test_input, test_arg))
                         if verbose > 0 and verbose < 3:

test-suite2/testcode/lib/testcode2/config.py

@@ -105,7 +105,8 @@ def parse_userconfig(config_file, executables=None, test_id=None,
     test_program_options = ('run_cmd_template',
         'launch_parallel', 'ignore_fields', 'data_tag', 'extract_cmd_template',
         'extract_fn', 'extract_program', 'extract_args', 'extract_fmt',
-        'verify', 'vcs', 'skip_program', 'skip_args', 'skip_cmd_template')
+        'verify', 'vcs', 'skip_program', 'skip_args', 'skip_cmd_template',
+        'can_fail')
     default_test_options = ('inputs_args', 'output', 'nprocs',
         'min_nprocs', 'max_nprocs', 'submit_template',)
     test_programs = {}
@@ -171,6 +172,10 @@ def parse_userconfig(config_file, executables=None, test_id=None,
         if 'vcs' in tp_dict:
             tp_dict['vcs'] = vcs.VCSRepository(tp_dict['vcs'],
                     os.path.dirname(exe))
+        if 'can_fail' in tp_dict:
+            tp_dict['can_fail'] = \
+                    userconfig.getboolean(section, 'can_fail')
+
         program = testcode2.TestProgram(section, exe, test_id,
             user_options['benchmark'], **tp_dict)
         test_programs[section] = program

test-suite2/tests/userconfig

@@ -25,6 +25,7 @@ tolerance = ( (1.0e-5, 1.0e-5, 'final_centres_x'),
 exe = ../../wannier90.x
 extract_fn = tools parsers.parse_werr.parse
 tolerance = ( (0.1, 0.01, 'warning_msg'))
+can_fail = true
 
 [WANNIER90_NNKP_OK]
 exe = ../../wannier90.x