Skip to content

Commit

Permalink
Filter the tests results by outcome (pytest-dev#46)
Browse files Browse the repository at this point in the history 
Fixes issue pytest-dev#38
  • Loading branch information
@RibeiroAna @davehunt
RibeiroAna authored and davehunt committed Jun 20, 2016
1 parent ebe6280 commit fb4d594
Show file tree
Hide file tree
Showing 3 changed files with 130 additions and 80 deletions.
88 changes: 65 additions & 23 deletions pytest_html/plugin.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,18 @@ def _appendrow(self, result, report):
if report.when != 'call':
test_id = '::'.join([report.nodeid, report.when])

self.test_logs.append(html.tr([
rows_table = html.tr([
html.td(result, class_='col-result'),
html.td(test_id, class_='col-name'),
html.td('{0:.2f}'.format(time), class_='col-duration'),
html.td(links_html, class_='col-links'),
html.td(additional_html, class_='extra')],
class_=result.lower() + ' results-table-row'))
html.td(links_html, class_='col-links')])

rows_extra = html.tr(html.td(additional_html,
class_='extra', colspan='5'))

self.test_logs.append(html.tbody(rows_table, rows_extra,
class_=result.lower() +
' results-table-row'))

def append_pass(self, report):
self.passed += 1
Expand Down Expand Up @@ -218,23 +223,56 @@ def pytest_sessionfinish(self, session):
html.title('Test Report'),
html.style(raw(style_css)))

class Outcome:

def __init__(self, outcome, total=0, label=None,
test_result=None, class_html=None):
self.outcome = outcome
self.label = label or outcome
self.class_html = class_html or outcome
self.total = total
self.test_result = test_result or outcome

self.generate_checkbox()
self.generate_summary_item()

def generate_checkbox(self):
checkbox_kwargs = {'data-test-result':
self.test_result.lower()}
if self.total == 0:
checkbox_kwargs['disabled'] = 'true'

self.checkbox = html.input(type='checkbox',
checked='true',
onChange='filter_table(this)',
name='filter_checkbox',
**checkbox_kwargs)

def generate_summary_item(self):
self.summary_item = html.span('{0} {1}'.
format(self.total, self.label),
class_=self.class_html)

outcomes = [Outcome('passed', self.passed),
Outcome('skipped', self.skipped),
Outcome('failed', self.failed),
Outcome('error', self.errors, label='errors'),
Outcome('xfailed', self.xfailed,
label='expected failures',
class_html='skipped'),
Outcome('xpassed', self.xpassed,
label='unexpected passes',
class_html='failed')]

summary = [html.h2('Summary'), html.p(
'{0} tests ran in {1:.2f} seconds.'.format(
numtests, suite_time_delta),
html.br(),
html.span('{0} passed'.format(
self.passed), class_='passed'), ', ',
html.span('{0} skipped'.format(
self.skipped), class_='skipped'), ', ',
html.span('{0} failed'.format(
self.failed), class_='failed'), ', ',
html.span('{0} errors'.format(
self.errors), class_='error'), '.',
html.br(),
html.span('{0} expected failures'.format(
self.xfailed), class_='skipped'), ', ',
html.span('{0} unexpected passes'.format(
self.xpassed), class_='failed'), '.')]
'{0} tests ran in {1:.2f} seconds. '.format(
numtests, suite_time_delta)),
html.p('(Un)check the boxes to filter the results.')]

for outcome in outcomes:
summary.append(outcome.checkbox)
summary.append(outcome.summary_item)
summary.append(' ')

results = [html.h2('Results'), html.table([html.thead(
html.tr([
Expand All @@ -245,9 +283,13 @@ def pytest_sessionfinish(self, session):
html.th('Duration',
class_='sortable numeric',
col='duration'),
html.th('Links')]), id='results-table-head'),
html.tbody(*self.test_logs, id='results-table-body')],
id='results-table')]
html.th('Links')]),
html.tr([
html.th('No results found. Try to check the filters',
colspan='5')],
id='not-found-message', hidden='true'),
id='results-table-head'),
self.test_logs], id='results-table')]

main_js = pkg_resources.resource_string(
__name__, os.path.join('resources', 'main.js'))
Expand Down
57 changes: 21 additions & 36 deletions pytest_html/resources/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this file,
* You can obtain one at http://mozilla.org/MPL/2.0/. */


function toArray(iter) {
if (iter === null) {
return null;
Expand Down Expand Up @@ -40,7 +41,6 @@ function sort_column(elem) {
addEventListener("DOMContentLoaded", function() {
reset_sort_headers();

split_extra_onto_two_rows();
sort_column(find('.initial-sort'));

find_all('.col-links a.image').forEach(function(elem) {
Expand Down Expand Up @@ -79,18 +79,14 @@ addEventListener("DOMContentLoaded", function() {
});

function sort_table(clicked, key_func) {
one_row_for_data();
var rows = find_all('.results-table-row');
var reversed = !clicked.classList.contains('asc');

var sorted_rows = sort(rows, key_func, reversed);

var parent = document.getElementById('results-table-body');
var parent = document.getElementById('results-table');
sorted_rows.forEach(function(elem) {
parent.appendChild(elem);
});

split_extra_onto_two_rows();
}

function sort(items, key_func, reversed) {
Expand All @@ -113,21 +109,21 @@ function sort(items, key_func, reversed) {

function key_alpha(col_index) {
return function(elem) {
return elem.childNodes[col_index].firstChild.data.toLowerCase();
return elem.childNodes[1].childNodes[col_index].firstChild.data.toLowerCase();
};
}

function key_num(col_index) {
return function(elem) {
return parseFloat(elem.childNodes[col_index].firstChild.data);
return parseFloat(elem.childNodes[1].childNodes[col_index].firstChild.data);
};
}

function key_result(col_index) {
return function(elem) {
var strings = ['Error', 'Failed', 'XFailed', 'XPassed', 'Skipped',
'Passed'];
return strings.indexOf(elem.childNodes[col_index].firstChild.data);
return strings.indexOf(elem.childNodes[1].childNodes[col_index].firstChild.data);
};
}

Expand Down Expand Up @@ -160,33 +156,22 @@ function toggle_sort_states(elem) {
}
}

function split_extra_onto_two_rows() {
find_all('tr.results-table-row').forEach(function(elem) {
var new_row = document.createElement("tr")
new_row.className = "extra";
elem.parentNode.insertBefore(new_row, elem.nextSibling);
find_all(".extra", elem).forEach(function (td_elem) {
if (find("*:not(.empty)", td_elem)) {
new_row.appendChild(td_elem);
td_elem.colSpan=5;
} else {
td_elem.parentNode.removeChild(td_elem);
}
});
});
function is_all_rows_hidden(value) {
return value.hidden == false;
}

function one_row_for_data() {
find_all('tr.results-table-row').forEach(function(elem) {
if (elem.nextSibling.classList.contains('extra')) {
toArray(elem.nextSibling.childNodes).forEach(
function (td_elem) {
elem.appendChild(td_elem);
})
} else {
var new_td = document.createElement("td");
new_td.className = "extra";
elem.appendChild(new_td);
}
});
function filter_table(elem) {
var outcome_att = "data-test-result";
var outcome = elem.getAttribute(outcome_att);
class_outcome = outcome + " results-table-row";
var outcome_rows = document.getElementsByClassName(class_outcome);

for(var i = 0; i < outcome_rows.length; i++){
outcome_rows[i].hidden = !elem.checked;
}

var rows = find_all('.results-table-row').filter(is_all_rows_hidden);
var all_rows_hidden = rows.length == 0 ? true : false;
var not_found_message = document.getElementById("not-found-message");
not_found_message.hidden = !all_rows_hidden;
}
65 changes: 44 additions & 21 deletions test_pytest_html.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,22 +25,45 @@ def run(testdir, path='report.html', *args):
return result, html


def assert_summary(html, tests=1, duration=None, passed=1, skipped=0, failed=0,
def assert_results_by_outcome(html, test_outcome, test_outcome_number,
label=None):
# Asserts if the test number of this outcome in the summary is correct
regex_summary = '(\d)+ {0}'.format(label or test_outcome)
assert int(re.search(regex_summary, html).group(1)) == test_outcome_number

# Asserts if the generated checkbox of this outcome is correct
regex_checkbox = ('<input checked="true" data-test-result="{0}"'
.format(test_outcome))
if test_outcome_number == 0:
regex_checkbox += ' disabled="true"'
assert re.search(regex_checkbox, html) is not None

# Asserts if the table rows of this outcome are correct
regex_table = ('tbody class=\"{0} '.format(test_outcome))
assert len(re.findall(regex_table, html)) == test_outcome_number


def assert_results(html, tests=1, duration=None, passed=1, skipped=0, failed=0,
errors=0, xfailed=0, xpassed=0):
m = re.search('(\d)+ tests ran in ([\d,.])+ seconds', html)
assert int(m.group(1)) == tests
# Asserts total amount of tests
total_tests = re.search('(\d)+ tests ran', html)
assert int(total_tests.group(1)) == tests

# Asserts tests running duration
if duration is not None:
assert float(m.group(2)) >= float(duration)
assert int(re.search('(\d)+ passed', html).group(1)) == passed
assert int(re.search('(\d)+ skipped', html).group(1)) == skipped
assert int(re.search('(\d)+ failed', html).group(1)) == failed
assert int(re.search('(\d)+ errors', html).group(1)) == errors
assert int(re.search('(\d)+ expected failures', html).group(1)) == xfailed
assert int(re.search('(\d)+ unexpected passes', html).group(1)) == xpassed
tests_duration = re.search('([\d,.])+ seconds', html)
assert float(tests_duration.group(1)) >= float(duration)

# Asserts by outcome
assert_results_by_outcome(html, 'passed', passed)
assert_results_by_outcome(html, 'skipped', skipped)
assert_results_by_outcome(html, 'failed', failed)
assert_results_by_outcome(html, 'error', errors, 'errors')
assert_results_by_outcome(html, 'xfailed', xfailed, 'expected failures')
assert_results_by_outcome(html, 'xpassed', xpassed, 'unexpected passes')

class TestHTML:

class TestHTML:
def test_durations(self, testdir):
sleep = float(0.2)
testdir.makepyfile("""
Expand All @@ -50,7 +73,7 @@ def test_sleep():
""".format(sleep))
result, html = run(testdir)
assert result.ret == 0
assert_summary(html, duration=sleep)
assert_results(html, duration=sleep)
p = re.compile('<td class="col-duration">([\d,.]+)</td>')
m = p.search(html)
assert float(m.group(1)) >= sleep
Expand All @@ -59,7 +82,7 @@ def test_pass(self, testdir):
testdir.makepyfile('def test_pass(): pass')
result, html = run(testdir)
assert result.ret == 0
assert_summary(html)
assert_results(html)

def test_skip(self, testdir):
reason = str(random.random())
Expand All @@ -70,14 +93,14 @@ def test_skip():
""".format(reason))
result, html = run(testdir)
assert result.ret == 0
assert_summary(html, tests=0, passed=0, skipped=1)
assert_results(html, tests=0, passed=0, skipped=1)
assert 'Skipped: {0}'.format(reason) in html

def test_fail(self, testdir):
testdir.makepyfile('def test_fail(): assert False')
result, html = run(testdir)
assert result.ret
assert_summary(html, passed=0, failed=1)
assert_results(html, passed=0, failed=1)
assert 'AssertionError' in html

def test_conditional_xfails(self, testdir):
Expand All @@ -94,7 +117,7 @@ def test_xpass(): pass
""")
result, html = run(testdir)
assert result.ret
assert_summary(html, tests=4, passed=1, failed=1, xfailed=1, xpassed=1)
assert_results(html, tests=4, passed=1, failed=1, xfailed=1, xpassed=1)

def test_setup_error(self, testdir):
testdir.makepyfile("""
Expand All @@ -105,7 +128,7 @@ def test_function(arg):
""")
result, html = run(testdir)
assert result.ret
assert_summary(html, tests=0, passed=0, errors=1)
assert_results(html, tests=0, passed=0, errors=1)
assert '::setup' in html
assert 'ValueError' in html

Expand All @@ -118,7 +141,7 @@ def test_xfail():
""".format(reason))
result, html = run(testdir)
assert result.ret == 0
assert_summary(html, passed=0, xfailed=1)
assert_results(html, passed=0, xfailed=1)
assert 'XFailed: {0}'.format(reason) in html

def test_xpass(self, testdir):
Expand All @@ -130,14 +153,14 @@ def test_xpass():
""")
result, html = run(testdir)
assert result.ret == 0
assert_summary(html, passed=0, xpassed=1)
assert_results(html, passed=0, xpassed=1)

def test_create_report_path(self, testdir):
testdir.makepyfile('def test_pass(): pass')
path = os.path.join('directory', 'report.html')
result, html = run(testdir, path)
assert result.ret == 0
assert_summary(html)
assert_results(html)

def test_resources(self, testdir):
testdir.makepyfile('def test_pass(): pass')
Expand Down Expand Up @@ -350,4 +373,4 @@ def test_foo(val):
""")
result, html = run(testdir)
assert result.ret == 0
assert_summary(html, passed=1)
assert_results(html, passed=1)

0 comments on commit fb4d594

Please sign in to comment.