Skip to content

Commit

Permalink
Stop attaching test reruns to final test report entries (pytest-dev#387)
Browse files Browse the repository at this point in the history
* treat rerun entries as seperate test runs

* Add changelog entry

* fix a typo

* remove debug code

* fix flaky test on mac
  • Loading branch information
gnikonorov authored Dec 1, 2020
1 parent 3f63686 commit 2bb2010
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 52 deletions.
4 changes: 4 additions & 0 deletions CHANGES.rst
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ Release Notes

**3.1.0 (unreleased)**

* Stop attaching test reruns to final test report entries (`#374 <https://github.com/pytest-dev/pytest-html/issues/374>`_)

* Thanks to `@VladimirPodolyan <https://github.com/VladimirPodolyan>`_ for reporting and `@gnikonorov <https://github.com/gnikonorov>`_ for the fix

* Allow for report duration formatting (`#376 <https://github.com/pytest-dev/pytest-html/issues/376>`_)

* Thanks to `@brettnolan <https://github.com/brettnolan>`_ for reporting and `@gnikonorov <https://github.com/gnikonorov>`_ for the fix
Expand Down
101 changes: 51 additions & 50 deletions src/pytest_html/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def __init__(self, outcome, report, logfile, config):
if getattr(report, "when", "call") != "call":
self.test_id = "::".join([report.nodeid, report.when])
self.time = getattr(report, "duration", 0.0)
self.formatted_time = getattr(report, "formatted_duration", 0.0)
self.formatted_time = self._format_time(report)
self.outcome = outcome
self.additional_html = []
self.links_html = []
Expand Down Expand Up @@ -283,6 +283,37 @@ def append_extra_html(self, extra, extra_index, test_index):
)
self.links_html.append(" ")

def _format_time(self, report):
# parse the report duration into its display version and return
# it to the caller
duration = getattr(report, "duration", None)
if duration is None:
return ""

duration_formatter = getattr(report, "duration_formatter", None)
string_duration = str(duration)
if duration_formatter is None:
if "." in string_duration:
split_duration = string_duration.split(".")
split_duration[1] = split_duration[1][0:2]

string_duration = ".".join(split_duration)

return string_duration
else:
# support %f, since time.strftime doesn't support it out of the box
# keep a precision of 2 for legacy reasons
formatted_milliseconds = "00"
if "." in string_duration:
milliseconds = string_duration.split(".")[1]
formatted_milliseconds = milliseconds[0:2]

duration_formatter = duration_formatter.replace(
"%f", formatted_milliseconds
)
duration_as_gmtime = time.gmtime(report.duration)
return time.strftime(duration_formatter, duration_as_gmtime)

def _populate_html_log_div(self, log, report):
if report.longrepr:
# longreprtext is only filled out on failure by pytest
Expand Down Expand Up @@ -425,6 +456,10 @@ def append_failed(self, report):
self.errors += 1
self._appendrow("Error", report)

def append_rerun(self, report):
self.rerun += 1
self._appendrow("Rerun", report)

def append_skipped(self, report):
if hasattr(report, "wasxfail"):
self.xfailed += 1
Expand All @@ -433,11 +468,6 @@ def append_skipped(self, report):
self.skipped += 1
self._appendrow("Skipped", report)

def append_other(self, report):
# For now, the only "other" the plugin give support is rerun
self.rerun += 1
self._appendrow("Rerun", report)

def _generate_report(self, session):
suite_stop_time = time.time()
suite_time_delta = suite_stop_time - self.suite_start_time
Expand Down Expand Up @@ -604,32 +634,6 @@ def generate_summary_item(self):
unicode_doc = unicode_doc.encode("utf-8", errors="xmlcharrefreplace")
return unicode_doc.decode("utf-8")

def _format_duration(self, report):
# parse the report duration into its display version and return it to the caller
duration_formatter = getattr(report, "duration_formatter", None)
string_duration = str(report.duration)
if duration_formatter is None:
if "." in string_duration:
split_duration = string_duration.split(".")
split_duration[1] = split_duration[1][0:2]

string_duration = ".".join(split_duration)

return string_duration
else:
# support %f, since time.strftime doesn't support it out of the box
# keep a precision of 2 for legacy reasons
formatted_milliseconds = "00"
if "." in string_duration:
milliseconds = string_duration.split(".")[1]
formatted_milliseconds = milliseconds[0:2]

duration_formatter = duration_formatter.replace(
"%f", formatted_milliseconds
)
duration_as_gmtime = time.gmtime(report.duration)
return time.strftime(duration_formatter, duration_as_gmtime)

def _generate_environment(self, config):
if not hasattr(config, "_metadata") or config._metadata is None:
return []
Expand Down Expand Up @@ -685,22 +689,23 @@ def _post_process_reports(self):
# through them all to figure out the outcome, xfail, duration,
# extras, and when it swapped from pass
for test_report in test_reports:
full_text += test_report.longreprtext
extras.extend(getattr(test_report, "extra", []))
duration += getattr(test_report, "duration", 0.0)
if test_report.outcome == "rerun":
# reruns are separate test runs for all intensive purposes
self.append_rerun(test_report)
else:
full_text += test_report.longreprtext
extras.extend(getattr(test_report, "extra", []))
duration += getattr(test_report, "duration", 0.0)

if (
test_report.outcome not in ("passed", "rerun")
and outcome == "passed"
):
outcome = test_report.outcome
failure_when = test_report.when
if (
test_report.outcome not in ("passed", "rerun")
and outcome == "passed"
):
outcome = test_report.outcome
failure_when = test_report.when

if hasattr(test_report, "wasxfail"):
wasxfail = True

if test_report.outcome == "rerun":
self.append_other(test_report)
if hasattr(test_report, "wasxfail"):
wasxfail = True

# the following test_report.<X> = settings come at the end of us
# looping through all test_reports that make up a single
Expand All @@ -715,7 +720,6 @@ def _post_process_reports(self):
test_report.longrepr = full_text
test_report.extra = extras
test_report.duration = duration
test_report.formatted_duration = self._format_duration(test_report)

if wasxfail:
test_report.wasxfail = True
Expand All @@ -728,9 +732,6 @@ def _post_process_reports(self):
test_report.when = failure_when
self.append_failed(test_report)

# we don't append other here since the only case supported
# for append_other is rerun, which is handled in the loop above

def pytest_runtest_logreport(self, report):
self.reports[report.nodeid].append(report)

Expand Down
34 changes: 32 additions & 2 deletions testing/test_pytest_html.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,17 +187,47 @@ def test_fail(self, testdir):
assert "AssertionError" in html

def test_rerun(self, testdir):
testdir.makeconftest(
"""
import pytest
@pytest.hookimpl(hookwrapper=True)
def pytest_runtest_makereport(item, call):
pytest_html = item.config.pluginmanager.getplugin("html")
outcome = yield
report = outcome.get_result()
extra = getattr(report, "extra", [])
if report.when == "call":
extra.append(pytest_html.extras.url("http://www.example.com/"))
report.extra = extra
"""
)

testdir.makepyfile(
"""
import pytest
@pytest.mark.flaky(reruns=5)
import time
@pytest.mark.flaky(reruns=2)
def test_example():
time.sleep(1)
assert False
"""
)

result, html = run(testdir)
assert result.ret
assert_results(html, passed=0, failed=1, rerun=5)
assert_results(html, passed=0, failed=1, rerun=2)

expected_report_durations = r'<td class="col-duration">1.\d{2}</td>'
assert len(re.findall(expected_report_durations, html)) == 3

expected_report_extras = (
r'<td class="col-links"><a class="url" href="http://www.example.com/" '
'target="_blank">URL</a> </td>'
)
assert len(re.findall(expected_report_extras, html)) == 3

def test_no_rerun(self, testdir):
testdir.makepyfile("def test_pass(): pass")
Expand Down

0 comments on commit 2bb2010

Please sign in to comment.