Skip to content

Commit

Permalink
scripts: twister: Fix NOTRUN in test_only
Browse files Browse the repository at this point in the history 
When using the --build-only into --test-only
Twister setup, NOTRUN statuses were not properly rerun.

Now they are properly run again if runnable.

Signed-off-by: Lukasz Mrugala <[email protected]>
  • Loading branch information
@LukaszMrugala @mmahadevan108
LukaszMrugala authored and mmahadevan108 committed Oct 31, 2024
1 parent 1e5a537 commit 9dc0af5
Show file tree
Hide file tree
Showing 5 changed files with 26 additions and 14 deletions.
6 changes: 6 additions & 0 deletions scripts/pylib/twister/twisterlib/reports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,6 +354,12 @@ def json_report(self, filename, version="NA", platform=None, filters=None):
elif instance.status == TwisterStatus.SKIP:
suite["status"] = TwisterStatus.SKIP
suite["reason"] = instance.reason
elif instance.status == TwisterStatus.NOTRUN:
suite["status"] = TwisterStatus.NOTRUN
suite["reason"] = instance.reason
else:
suite["status"] = TwisterStatus.NONE
suite["reason"] = 'Unknown Instance status.'

if instance.status != TwisterStatus.NONE:
suite["execution_time"] = f"{float(handler_time):.2f}"
Expand Down
9 changes: 6 additions & 3 deletions scripts/pylib/twister/twisterlib/testplan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -659,6 +659,9 @@ def load_from_file(self, file, filter_platform=[]):
self.hwm
)

if self.options.test_only and not instance.run:
continue

instance.metrics['handler_time'] = ts.get('execution_time', 0)
instance.metrics['used_ram'] = ts.get("used_ram", 0)
instance.metrics['used_rom'] = ts.get("used_rom",0)
Expand All @@ -676,9 +679,9 @@ def load_from_file(self, file, filter_platform=[]):
instance.status = TwisterStatus.NONE
instance.reason = None
instance.retries += 1
# test marked as passed (built only) but can run when
# --test-only is used. Reset status to capture new results.
elif status == TwisterStatus.PASS and instance.run and self.options.test_only:
# test marked as built only can run when --test-only is used.
# Reset status to capture new results.
elif status == TwisterStatus.NOTRUN and instance.run and self.options.test_only:
instance.status = TwisterStatus.NONE
instance.reason = None
else:
Expand Down
3 changes: 2 additions & 1 deletion scripts/tests/twister/test_testplan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -766,6 +766,7 @@ def test_testplan_load(
testplan.apply_filters = mock.Mock()

with mock.patch('twisterlib.testinstance.TestInstance.create_overlay', mock.Mock()), \
mock.patch('twisterlib.testinstance.TestInstance.check_runnable', return_value=True), \
pytest.raises(exception) if exception else nullcontext():
testplan.load()

Expand Down Expand Up @@ -1600,7 +1601,7 @@ def get_platform(name):
'testcases': {
'TS1.tc1': {
'status': TwisterStatus.PASS,
'reason': None,
'reason': 'passed',
'duration': 60.0,
'output': ''
}
Expand Down
16 changes: 9 additions & 7 deletions scripts/tests/twister_blackbox/test_footprint.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

# pylint: disable=no-name-in-module
from conftest import ZEPHYR_BASE, TEST_DATA, testsuite_filename_mock, clear_log_in_test
from twisterlib.statuses import TwisterStatus
from twisterlib.testplan import TestPlan


Expand Down Expand Up @@ -76,9 +77,10 @@ def test_compare_report(self, caplog, out_path, old_ram_multiplier, expect_delta
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
# We assume positive RAM usage.
ts[self.RAM_KEY] *= old_ram_multiplier

with open(os.path.join(out_path, 'twister.json'), 'w') as f:
f.write(json.dumps(j, indent=4))

Expand Down Expand Up @@ -137,7 +139,7 @@ def test_footprint_from_buildlog(self, out_path):
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
assert self.RAM_KEY in ts
old_values += [ts[self.RAM_KEY]]

Expand All @@ -162,7 +164,7 @@ def test_footprint_from_buildlog(self, out_path):
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
assert self.RAM_KEY in ts
new_values += [ts[self.RAM_KEY]]

Expand Down Expand Up @@ -202,7 +204,7 @@ def test_footprint_threshold(self, caplog, out_path, old_ram_multiplier,
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
# We assume positive RAM usage.
ts[self.RAM_KEY] *= old_ram_multiplier
with open(os.path.join(out_path, 'twister.json'), 'w') as f:
Expand Down Expand Up @@ -271,7 +273,7 @@ def test_show_footprint(self, caplog, out_path, flags, old_ram_multiplier, expec
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
# We assume positive RAM usage.
ts[self.RAM_KEY] *= old_ram_multiplier
with open(os.path.join(out_path, 'twister.json'), 'w') as f:
Expand Down Expand Up @@ -344,7 +346,7 @@ def test_last_metrics(self, caplog, out_path, old_ram_multiplier, expect_delta_l
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
# We assume positive RAM usage.
ts[self.RAM_KEY] *= old_ram_multiplier
with open(os.path.join(out_path, 'twister.json'), 'w') as f:
Expand Down Expand Up @@ -441,7 +443,7 @@ def test_all_deltas(self, caplog, out_path, old_ram_multiplier, expect_delta_log
with open(os.path.join(out_path, 'twister.json')) as f:
j = json.load(f)
for ts in j['testsuites']:
if 'reason' not in ts:
if TwisterStatus(ts.get('status')) == TwisterStatus.NOTRUN:
# We assume positive RAM usage.
ts[self.RAM_KEY] *= old_ram_multiplier
with open(os.path.join(out_path, 'twister.json'), 'w') as f:
Expand Down
6 changes: 3 additions & 3 deletions scripts/tests/twister_blackbox/test_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,19 +46,19 @@ class TestRunner:
['qemu_x86/atom', 'qemu_x86_64/atom', 'intel_adl_crb/alder_lake'],
{
'selected_test_scenarios': 3,
'selected_test_instances': 6,
'selected_test_instances': 4,
'skipped_configurations': 0,
'skipped_by_static_filter': 0,
'skipped_at_runtime': 0,
'passed_configurations': 4,
'built_configurations': 2,
'built_configurations': 0,
'failed_configurations': 0,
'errored_configurations': 0,
'executed_test_cases': 8,
'skipped_test_cases': 0,
'platform_count': 0,
'executed_on_platform': 4,
'only_built': 2
'only_built': 0
}
)
]
Expand Down

0 comments on commit 9dc0af5

Please sign in to comment.