Skip to content

Commit

Permalink
Added deep links and full feature/scenario reporting to the emailable…
Browse files Browse the repository at this point in the history 
… reports
  • Loading branch information
@wakaleo
wakaleo committed Mar 10, 2019
1 parent ed7dc82 commit 5285e47
Show file tree
Hide file tree
Showing 7 changed files with 161 additions and 61 deletions.
1 change: 1 addition & 0 deletions serenity-emailer/src/main/kotlin/net/serenitybdd/reports/email/EmailReporter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ class EmailReporter(val environmentVariables: EnvironmentVariables) : ExtendedRe
minTestDuration = formattedDuration(minDurationOf(testOutcomes.outcomes))
),
"failuresByFeature" to FailuresByFeature.from(testOutcomes),
"resultsByFeature" to TestResultsByFeature.from(testOutcomes),
"frequentFailures" to FrequentFailures.from(testOutcomes).withMaxOf(scoreboardSize),
"unstableFeatures" to UnstableFeatures.from(testOutcomes).withMaxOf(scoreboardSize),
"coverage" to TagCoverage.from(testOutcomes).forTagTypes(tagTypes),
Expand Down
88 changes: 65 additions & 23 deletions serenity-emailer/src/main/resources/templates/email/serenity-summary-report-inlined.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,10 @@



.for-success {
color: #52B255;
}

.for-passing {
color: #52B255;
}
Expand Down Expand Up @@ -716,22 +720,8 @@
.failure-scoreboard th {
text-align: left;
}

















.failure-scoreboard tr:nth-child(even) {background: #f5f5f5}

table.count-badge {
padding: 0px;
Expand Down Expand Up @@ -1033,15 +1023,15 @@
</tr>
<tr th:block th:if="${!#strings.isEmpty(report.link)}">
<td class="compact-wrapper" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;box-sizing:border-box;padding-left:24px;padding-right:24px;padding-top:4px;padding-bottom:4px;" >
<a th:href="${report.link}">View full report</a>
<a style="text-transform: uppercase;color:#8accf2; text-decoration: none; font-weight: bold; padding: 0.5em 1em; background: #316d91;" th:href="${report.link}">View full report</a>
</td>
</tr>
<tr th:block th:if="${#lists.size(coverage) > 0}">
<td class="compact-wrapper" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;box-sizing:border-box;padding-left:24px;padding-right:24px;padding-top:4px;padding-bottom:4px;" ><h3 style="color:#222222;font-family:Helvetica, sans-serif;font-weight:400;line-height:1.4;margin-top:0;margin-bottom:0;margin-right:0;margin-left:0;font-size:20px;text-align:center;" >Functional Coverage</h3></td>
</tr>
<tr th:each="tagCoverageByType : ${coverage}">
<td class="compact-wrapper" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;box-sizing:border-box;padding-left:24px;padding-right:24px;padding-top:4px;padding-bottom:4px;" th:if="${#lists.size(tagCoverageByType.tagCoverage) > 0}">
<h4 class="tag-title" th:text="${tagCoverageByType.tagType}" style="color:#222222;font-family:Helvetica, sans-serif;line-height:1.4;margin-bottom:0;margin-right:0;margin-left:0;font-weight:500;font-size:18px;margin-top:20px;text-transform:capitalize;" >Control Creation</h4>
<h4 class="tag-title" th:text="${tagCoverageByType.tagTitle}" style="color:#222222;font-family:Helvetica, sans-serif;line-height:1.4;margin-bottom:0;margin-right:0;margin-left:0;font-weight:500;font-size:18px;margin-top:20px;text-transform:capitalize;" >Control Creation</h4>
<table class="test-results-table categories" style="mso-table-lspace:0pt;mso-table-rspace:0pt;width:100%;border-width:1px;border-style:solid;border-color:grey;margin-bottom:26px;border-collapse:collapse;" >
<tr>
<th width="60%" th:text="${report.tagCategoryTitle}">Category</th>
Expand Down Expand Up @@ -1174,19 +1164,71 @@ <h3 style="color:#222222;font-family:Helvetica, sans-serif;font-weight:400;line-
</tr>
<span th:each="failingFeature : ${failuresByFeature}">
<tr>
<td colspan="2" th:text=${failingFeature.featureName} class="feature" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;font-style:italic;" >A
Failing Feature
<td colspan="2" th:text=${failingFeature.featureName} class="feature" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;font-style:italic;padding-left:16px;" >
A Failing Feature
</td>
</tr>
<tr th:each="failingScenario: ${failingFeature.failures}">
<td class="scenarioName" th:text="${failingScenario.title}" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;padding-left:8px;min-width:4em;width:55%;" >Failing scenario</td>
<td th:text="${failingScenario.errorMessage}"
th:attr="class='frequent-failure for-' + ${failingScenario.result}" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;" >Error message</td>
<td class="scenarioName" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;padding-left:32px;min-width:4em;width:55%;" >
<a th:if="${!#strings.isEmpty(report.link)}" th:text="${failingScenario.title}" th:href="${#strings.concat(report.link,'/', failingScenario.reportName)}">
Failing scenario
</a>
<a th:unless="${!#strings.isEmpty(report.link)}" th:text="${failingScenario.title}"></a>
</td>
<td th:attr="class='frequent-failure for-' + ${failingScenario.result}" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;" >
<div th:text="${failingScenario.result}"
th:attr="class='frequent-failure for-' + ${failingScenario.result}"
class="scenario-summary-tag for-passing" style="font-weight: bold; text-transform: uppercase; ">ERROR</div>
<div th:text="${failingScenario.errorMessage}"
th:attr="class='frequent-failure for-' + ${failingScenario.result}"
class="frequent-failure for-error">Error message</div>
</td>
</tr>
</span>
</table>
</td>
</tr>

<!-- TEST RESULT LIST -->
<tr>
<td class="compact-wrapper" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;box-sizing:border-box;padding-left:24px;padding-right:24px;padding-top:4px;padding-bottom:4px;" >
<h3 style="color:#222222;font-family:Helvetica, sans-serif;font-weight:400;line-height:1.4;margin-top:0;margin-bottom:0;margin-right:0;margin-left:0;font-size:20px;text-align:center;" >Full Test Results</h3>
<table class="failure-list failure-scoreboard" style="border-width:1px;border-style:solid;border-color:#acb1b9;border-collapse:separate;mso-table-lspace:0pt;mso-table-rspace:0pt;width:100%;" >
<tr>
<th style="text-align:left;" >Requirement</th>
<th style="text-align:left;" >Result</th>
</tr>
<span th:each="feature : ${resultsByFeature}">
<tr>
<td colspan="2" th:text=${feature.featureName} class="feature feature-title" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;font-style:italic;padding-left:16px;" >A Feature
</td>
</tr>
<tr th:each="scenario: ${feature.scenarios}">
<td class="scenarioName" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;padding-left:32px;min-width:4em;width:55%;">
<a th:if="${!#strings.isEmpty(report.link)}" th:href="${#strings.concat(report.link,'/', scenario.reportName)}" th:text="${scenario.title}">Scenario</a>
<a th:unless="${!#strings.isEmpty(report.link)}" th:text="${scenario.title}">Scenario</a>

</td>
<td>
<div th:text="${scenario.result}"
th:attr="class='frequent-failure for-' + ${scenario.result}"
class="scenario-summary-tag for-passing" style="font-weight: bold; text-transform: uppercase; ">PASSING</div>
<div th:text="${scenario.errorMessage}"
th:attr="class='frequent-failure for-' + ${scenario.result}"
class="frequent-failure for-error">Error message</div>
<td>

<!--
<td th:text="${scenario.errorMessage}"
th:attr="class='frequent-failure for-' + ${scenario.result}" style="font-family:Helvetica, sans-serif;font-size:14px;vertical-align:top;" >Error message</td>
-->


</span>
</table>
</td>
</tr>

<tr class="footer" style="clear:both;padding-top:24px;text-align:center;width:100%;" >
<td style="font-family:Helvetica, sans-serif;vertical-align:top;color:#999999;font-size:12px;text-align:center;" >Report produced by Serenity BDD</td>
</tr>
Expand Down
69 changes: 48 additions & 21 deletions ...-emailer/src/test/kotlin/net/serenitybdd/reports/email/WhenGeneratingAnEmailableReport.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import java.nio.file.Paths
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class WhenGeneratingAnEmailableReport {

val TEST_OUTCOMES_WITH_A_SINGLE_TEST = ClassLoader.getSystemResource("test_outcomes/with_different_results").path
val TEST_OUTCOMES_WITH_MULTIPLE_RESULTS = ClassLoader.getSystemResource("test_outcomes/with_different_results").path

private val environmentVariables: EnvironmentVariables = MockEnvironmentVariables()

Expand All @@ -32,7 +32,7 @@ class WhenGeneratingAnEmailableReport {
private val reportContents: String

init {
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_A_SINGLE_TEST))
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
}

Expand Down Expand Up @@ -73,21 +73,21 @@ class WhenGeneratingAnEmailableReport {
environmentVariables.setProperty("report.customfields.User", "tim")
environmentVariables.setProperty("report.customfields.order", "environment,version,HostName,User")

generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_A_SINGLE_TEST))
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
parsedReport = parse(reportContents)
}

@Test
fun `should get display customisable environment variables from the report-summary-* properties`() {
val fieldValues = parsedReport.getElementsByClass("custom-value").map { element -> element.text() }
assertThat(fieldValues).contains("NAV Automation INT5","INT NAV 13.5.0","localhost","tim")
assertThat(fieldValues).contains("NAV Automation INT5", "INT NAV 13.5.0", "localhost", "tim")
}

@Test
fun `customisable environment variables should appear in the order specified in the report-dot-customfields-order field`() {
val fieldValues = parsedReport.getElementsByClass("custom-title").map { element -> element.text() }
assertThat(fieldValues).contains("Environment","Version","Host name","User")
assertThat(fieldValues).contains("Environment", "Version", "Host name", "User")
}
}

Expand All @@ -100,28 +100,28 @@ class WhenGeneratingAnEmailableReport {
init {
environmentVariables.setProperty("report.tagtypes", "group, feature")

generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_A_SINGLE_TEST))
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
parsedReport = parse(reportContents)
}

@Test
fun `should list all configured summary tag types as headings`() {
val tagTitles = parsedReport.getElementsByClass("tag-title").map { element -> element.text() }
assertThat(tagTitles).containsExactly("Group","Feature")
assertThat(tagTitles).containsExactly("Group", "Feature")
}

@Test
fun `should list the tags of each specified tag type as sub-headings`() {
val tagSubTitles = parsedReport.getElementsByClass("tag-subtitle").map { element -> element.text() }
assertThat(tagSubTitles).contains("Alpha","Beta","Gamma")
assertThat(tagSubTitles).contains("Alpha", "Beta", "Gamma")
}

@Test
fun `should list feature tags in their shortened form`() {
val tagSubTitles = parsedReport.getElementsByClass("tag-subtitle").map { element -> element.text() }
assertThat(tagSubTitles).contains("Broken scenarios","Compromised scenarios","Failed scenarios","Ignored scenarios",
"Mixed scenarios","Passing scenarios","Pending scenarios")
assertThat(tagSubTitles).contains("Broken scenarios", "Compromised scenarios", "Failed scenarios", "Ignored scenarios",
"Mixed scenarios", "Passing scenarios", "Pending scenarios")
}

@Test
Expand All @@ -130,12 +130,13 @@ class WhenGeneratingAnEmailableReport {
.getElementsByClass("frequent-failure")
.map { it.text() }

assertThat(unstableFeatures).containsExactly("Assertion error","Illegal argument exception") }
assertThat(unstableFeatures).containsExactly("Assertion error", "Illegal argument exception")
}

@Test
fun `should list top most unstable features`() {
val unstableFeatures = parsedReport.getElementsByClass("unstable-feature").map { element -> element.text() }
assertThat(unstableFeatures).containsExactly("Broken scenarios","Failed scenarios","Mixed scenarios")
assertThat(unstableFeatures).containsExactly("Broken scenarios", "Failed scenarios", "Mixed scenarios")
}
}

Expand All @@ -146,7 +147,7 @@ class WhenGeneratingAnEmailableReport {
private val parsedReport: Document

init {
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_A_SINGLE_TEST))
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
parsedReport = parse(reportContents)
}
Expand All @@ -155,22 +156,22 @@ class WhenGeneratingAnEmailableReport {
@Test
fun `should list feature tags in their shortened form`() {
val tagSubTitles = parsedReport.getElementsByClass("tag-subtitle").map { element -> element.text() }
assertThat(tagSubTitles).contains("Broken scenarios","Compromised scenarios","Failed scenarios","Ignored scenarios",
"Mixed scenarios","Passing scenarios","Pending scenarios")
assertThat(tagSubTitles).contains("Broken scenarios", "Compromised scenarios", "Failed scenarios", "Ignored scenarios",
"Mixed scenarios", "Passing scenarios", "Pending scenarios")
}

@Test
fun `should list top most frequent failures features`() {
val unstableFeatures = parsedReport.getElementsByClass("failure-scoreboard")[0]
.getElementsByClass("frequent-failure")
.map { it.text() }
assertThat(unstableFeatures).containsExactly("Assertion error","Illegal argument exception")
.getElementsByClass("frequent-failure")
.map { it.text() }
assertThat(unstableFeatures).containsExactly("Assertion error", "Illegal argument exception")
}

@Test
fun `should list top most unstable features`() {
val unstableFeatures = parsedReport.getElementsByClass("unstable-feature").map { element -> element.text() }
assertThat(unstableFeatures).containsExactly("Broken scenarios","Failed scenarios","Mixed scenarios")
assertThat(unstableFeatures).containsExactly("Broken scenarios", "Failed scenarios", "Mixed scenarios")
}

@Nested
Expand All @@ -182,7 +183,7 @@ class WhenGeneratingAnEmailableReport {
init {
environmentVariables.setProperty("report.scoreboard.size", "2")

generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_A_SINGLE_TEST))
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
parsedReport = parse(reportContents)
}
Expand All @@ -195,7 +196,33 @@ class WhenGeneratingAnEmailableReport {
}
}
}

@Nested
inner class ReportShowingTheFullFeatureList {
private val generatedReport: File
private val reportContents: String
private val parsedReport: Document

init {
generatedReport = EmailReporter(environmentVariables).generateReportFrom(Paths.get(TEST_OUTCOMES_WITH_MULTIPLE_RESULTS))
reportContents = generatedReport.readText()
parsedReport = parse(reportContents)
}


@Test
fun `should list feature titles`() {
val featureTitles = parsedReport.getElementsByClass("feature-title").map { element -> element.text() }
assertThat(featureTitles).contains("Broken scenarios",
"Compromised scenarios",
"Failed scenarios",
"Ignored scenarios",
"Mixed scenarios",
"Passing scenarios",
"Pending scenarios")
}
}
}

fun parse(html: String) : Document = Jsoup.parse(html)
fun parse(html: String): Document = Jsoup.parse(html)

Loading

0 comments on commit 5285e47

Please sign in to comment.