twister: deal with predicates and skipped tests

Using ztest predicate feature, testcases are skipped, however, the skip
is only reported as part of the summary and not through normal
execution. Until now such tests were reported as blocked or had a null
status in the twister json output.

This changes will look into summary results and use them to confirm
parsed results and add any missing results that either were not reported
or not captured through the serial console.

The issue can be observed with the drivers.can.api test for example on
the frdm_k64f board.

Signed-off-by: Anas Nashif <[email protected]>

scripts/pylib/twister/twisterlib/harness.py

@@ -624,6 +624,12 @@ def handle(self, line):
             self._match = True
 
         result_match = result_re.match(line)
+        # some testcases are skipped based on predicates and do not show up
+        # during test execution, however they are listed in the summary. Parse
+        # the summary for status and use that status instead.
+
+        summary_re = re.compile(r"- (PASS|FAIL|SKIP) - \[([^\.]*).(test_)?(\S*)\] duration = (\d*[.,]?\d*) seconds")
+        summary_match = summary_re.match(line)
 
         if result_match:
             matched_status = result_match.group(1)
@@ -638,6 +644,20 @@ def handle(self, line):
             self.testcase_output = ""
             self._match = False
             self.ztest = True
+        elif summary_match:
+            matched_status = summary_match.group(1)
+            self.detected_suite_names.append(summary_match.group(2))
+            name = "{}.{}".format(self.id, summary_match.group(4))
+            tc = self.instance.get_case_or_create(name)
+            tc.status = self.ztest_to_status[matched_status]
+            if tc.status == "skipped":
+                tc.reason = "ztest skip"
+            tc.duration = float(summary_match.group(5))
+            if tc.status == "failed":
+                tc.output = self.testcase_output
+            self.testcase_output = ""
+            self._match = False
+            self.ztest = True
 
         self.process_test(line)