Skip to content

Commit

Permalink
CI Treat Errors As Warnings - ref pester#2295 (pester#2296)
Browse files Browse the repository at this point in the history
  • Loading branch information
@erikgraa
erikgraa authored Apr 25, 2023
1 parent 36b3164 commit 06ab106
Show file tree
Hide file tree
Showing 6 changed files with 167 additions and 13 deletions.
2 changes: 1 addition & 1 deletion src/Main.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1239,7 +1239,7 @@ function Invoke-Pester {

if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
$errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorMessage[0] -Message $errorMessage[1..($errorMessage.Count - 1)]
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -CILogLevel $PesterPreference.Output.CILogLevel.Value -Header $errorMessage[0] -Message $errorMessage[1..($errorMessage.Count - 1)]
}
else {
Write-ErrorToScreen @formatErrorParams -Throw:$PesterPreference.Run.Throw.Value
Expand Down
3 changes: 3 additions & 0 deletions src/Pester.RSpec.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,9 @@ function New-PesterConfiguration {
CIFormat: The CI format of error output in build logs, options are None, Auto, AzureDevops and GithubActions.
Default value: 'Auto'
CILogLevel: The CI log level in build logs, options are Error and Warning.
Default value: 'Error'
RenderMode: The mode used to render console output, options are Auto, Ansi, ConsoleColor and Plaintext.
Default value: 'Auto'
Expand Down
19 changes: 19 additions & 0 deletions src/csharp/Pester/OutputConfiguration.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ public class OutputConfiguration : ConfigurationSection
private StringOption _verbosity;
private StringOption _stackTraceVerbosity;
private StringOption _ciFormat;
private StringOption _ciLogLevel;
private StringOption _renderMode;

public static OutputConfiguration Default { get { return new OutputConfiguration(); } }
Expand All @@ -40,6 +41,7 @@ public OutputConfiguration(IDictionary configuration) : this()
configuration.AssignObjectIfNotNull<string>(nameof(Verbosity), v => Verbosity = v);
configuration.AssignObjectIfNotNull<string>(nameof(StackTraceVerbosity), v => StackTraceVerbosity = v);
configuration.AssignObjectIfNotNull<string>(nameof(CIFormat), v => CIFormat = v);
configuration.AssignObjectIfNotNull<string>(nameof(CILogLevel), v => CILogLevel = v);
configuration.AssignObjectIfNotNull<string>(nameof(RenderMode), v => RenderMode = v);
}
}
Expand All @@ -49,6 +51,7 @@ public OutputConfiguration() : base("Output configuration")
Verbosity = new StringOption("The verbosity of output, options are None, Normal, Detailed and Diagnostic.", "Normal");
StackTraceVerbosity = new StringOption("The verbosity of stacktrace output, options are None, FirstLine, Filtered and Full.", "Filtered");
CIFormat = new StringOption("The CI format of error output in build logs, options are None, Auto, AzureDevops and GithubActions.", "Auto");
CILogLevel = new StringOption("The CI log level in build logs, options are Error and Warning.", "Error");
RenderMode = new StringOption("The mode used to render console output, options are Auto, Ansi, ConsoleColor and Plaintext.", "Auto");
}

Expand Down Expand Up @@ -100,6 +103,22 @@ public StringOption CIFormat
}
}

public StringOption CILogLevel
{
get { return _ciLogLevel; }
set
{
if (_ciLogLevel == null)
{
_ciLogLevel = value;
}
else
{
_ciLogLevel = new StringOption(_ciLogLevel, value?.Value);
}
}
}

public StringOption RenderMode
{
get { return _renderMode; }
Expand Down
46 changes: 36 additions & 10 deletions src/functions/Output.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ function Write-PesterHostMessage {
$message = "$($message -replace '(?m)^', "$fg$bg")$($ANSIcodes.ResetAll)"

& $SafeCommands['Write-Host'] -Object $message -NoNewLine:$NoNewLine
} else {
}
else {
if ($RenderMode -eq 'Plaintext') {
if ($PSBoundParameters.ContainsKey('ForegroundColor')) {
$null = $PSBoundParameters.Remove('ForegroundColor')
Expand Down Expand Up @@ -627,7 +628,7 @@ function Get-WriteScreenPlugin ($Verbosity) {

if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
$errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -CILogLevel $PesterPreference.Output.CILogLevel.Value -Header $errorHeader -Message $errorMessage
}
else {
Write-PesterHostMessage -ForegroundColor $ReportTheme.Fail $errorHeader
Expand Down Expand Up @@ -700,7 +701,7 @@ function Get-WriteScreenPlugin ($Verbosity) {

if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
$errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -CILogLevel $PesterPreference.Output.CILogLevel.Value -Header $errorHeader -Message $errorMessage
}
else {
Write-PesterHostMessage -ForegroundColor $ReportTheme.Fail $errorHeader
Expand Down Expand Up @@ -775,6 +776,10 @@ function Get-WriteScreenPlugin ($Verbosity) {
throw "Unsupported CI format '$($PesterPreference.Output.CIFormat.Value)'"
}

if ($PesterPreference.Output.CILogLevel.Value -notin 'Error', 'Warning') {
throw "Unsupported CI log level '$($PesterPreference.Output.CILogLevel.Value)'"
}

$humanTime = "$(Get-HumanTime ($_test.Duration)) ($(Get-HumanTime $_test.UserDuration)|$(Get-HumanTime $_test.FrameworkDuration))"

if ($PesterPreference.Debug.ShowNavigationMarkers.Value) {
Expand Down Expand Up @@ -817,7 +822,7 @@ function Get-WriteScreenPlugin ($Verbosity) {

if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
$errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header "$margin[-] $out $humanTime" -Message $errorMessage
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -CILogLevel $PesterPreference.Output.CILogLevel.Value -Header "$margin[-] $out $humanTime" -Message $errorMessage
}
else {
Write-PesterHostMessage -ForegroundColor $ReportTheme.Fail "$margin[-] $out" -NoNewLine
Expand Down Expand Up @@ -920,7 +925,7 @@ function Get-WriteScreenPlugin ($Verbosity) {

if ($PesterPreference.Output.CIFormat.Value -in 'AzureDevops', 'GithubActions') {
$errorMessage = (Format-ErrorMessage @formatErrorParams) -split [Environment]::NewLine
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -Header $errorHeader -Message $errorMessage
Write-CIErrorToScreen -CIFormat $PesterPreference.Output.CIFormat.Value -CILogLevel $PesterPreference.Output.CILogLevel.Value -Header $errorHeader -Message $errorMessage
}
else {
Write-PesterHostMessage -ForegroundColor $ReportTheme.BlockFail $errorHeader
Expand All @@ -945,6 +950,10 @@ function Format-CIErrorMessage {
[ValidateSet('AzureDevops', 'GithubActions', IgnoreCase)]
[string] $CIFormat,

[Parameter(Mandatory)]
[ValidateSet('Error', 'Warning', IgnoreCase)]
[string] $CILogLevel,

[Parameter(Mandatory)]
[string] $Header,

Expand All @@ -962,20 +971,33 @@ function Format-CIErrorMessage {

# header task issue error, so it gets reported to build log
# https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=powershell#example-log-an-error
$headerTaskIssueError = "##vso[task.logissue type=error] $Header"
$lines.Add($headerTaskIssueError)
# https://learn.microsoft.com/en-us/azure/devops/pipelines/scripts/logging-commands?view=azure-devops&tabs=powershell#example-log-a-warning-about-a-specific-place-in-a-file
switch ($CILogLevel) {
"Error" { $logIssueType = 'error' }
"Warning" { $logIssueType = 'warning' }
Default { $logIssueType = 'error' }
}

$headerLoggingCommand = "##vso[task.logissue type=$logIssueType] $Header"
$lines.Add($headerLoggingCommand)

# Add subsequent messages as errors, but do not get reported to build log
foreach ($line in $Message) {
$lines.Add("##[error] $line")
$lines.Add("##[$logIssueType] $line")
}
}
elseif ($CIFormat -eq 'GithubActions') {

# header error, so it gets reported to build log
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message
$headerError = "::error::$($Header.TrimStart())"
$lines.Add($headerError)
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-a-warning-message
switch ($CILogLevel) {
"Error" { $headerWorkflowCommand = "::error::$($Header.TrimStart())" }
"Warning" { $headerWorkflowCommand = "::warning::$($Header.TrimStart())" }
Default { $headerWorkflowCommand = "::error::$($Header.TrimStart())" }
}

$lines.Add($headerWorkflowCommand)

# Add rest of messages inside expandable group
# https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#grouping-log-lines
Expand All @@ -998,6 +1020,10 @@ function Write-CIErrorToScreen {
[ValidateSet('AzureDevops', 'GithubActions', IgnoreCase)]
[string] $CIFormat,

[Parameter(Mandatory)]
[ValidateSet('Error', 'Warning', IgnoreCase)]
[string] $CILogLevel,

[Parameter(Mandatory)]
[string] $Header,

Expand Down
43 changes: 43 additions & 0 deletions tst/Pester.RSpec.Configuration.ts.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ i -PassThru:$PassThru {
[PesterConfiguration]::Default.Output.CIFormat.Value | Verify-Equal Auto
}

t "Output.CILogLevel is Error" {
[PesterConfiguration]::Default.Output.CILogLevel.Value | Verify-Equal 'Error'
}

t "Output.RenderMode is Auto" {
[PesterConfiguration]::Default.Output.RenderMode.Value | Verify-Equal 'Auto'
}
Expand Down Expand Up @@ -1118,6 +1122,45 @@ i -PassThru:$PassThru {
}
}

b "Output.CILogLevel" {
t "Each option can be set and updated" {
$c = [PesterConfiguration] @{
Run = @{
ScriptBlock = { }
PassThru = $true
}
}

foreach ($option in "Error", "Warning") {
$c.Output.CILogLevel = $option
$r = Invoke-Pester -Configuration $c
$r.Configuration.Output.CILogLevel.Value | Verify-Equal $option
}
}

t "Exception is thrown when incorrect option is set" {
$sb = {
Describe "a" {
It "b" {}
}
}

$c = [PesterConfiguration] @{
Run = @{
ScriptBlock = $sb
PassThru = $true
Throw = $true
}
Output = @{
CIFormat = 'None'
CILogLevel = 'Something'
}
}

{ Invoke-Pester -Configuration $c } | Verify-Throw
}
}

b 'Output.RenderMode' {
t 'Output.RenderMode is Plaintext when set to Auto (default) and env:NO_COLOR is set' {
$c = [PesterConfiguration] @{
Expand Down
67 changes: 65 additions & 2 deletions tst/functions/Output.Tests.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -624,7 +624,31 @@ InModuleScope -ModuleName Pester -ScriptBlock {
)
}
) {
Format-CIErrorMessage -CIFormat 'AzureDevops' -Header $Header -Message $Message | Should -Be $Expected
Format-CIErrorMessage -CIFormat 'AzureDevops' -CILogLevel 'Error' -Header $Header -Message $Message | Should -Be $Expected
}
}

Context "Azure Devops Warning Format" {
It "Header '<header>' and Message '<message>' returns '<expected>'" -TestCases @(
@{
Header = 'header'
Message = 'message'
Expected = @(
'##vso[task.logissue type=warning] header',
'##[warning] message'
)
}
@{
Header = 'header'
Message = @('message1', 'message2')
Expected = @(
'##vso[task.logissue type=warning] header',
'##[warning] message1',
'##[warning] message2'
)
}
) {
Format-CIErrorMessage -CIFormat 'AzureDevops' -CILogLevel 'Warning' -Header $Header -Message $Message | Should -Be $Expected
}
}

Expand Down Expand Up @@ -663,7 +687,46 @@ InModuleScope -ModuleName Pester -ScriptBlock {
)
}
) {
Format-CIErrorMessage -CIFormat 'GithubActions' -Header $Header -Message $Message | Should -Be $Expected
Format-CIErrorMessage -CIFormat 'GithubActions' -CILogLevel 'Error' -Header $Header -Message $Message | Should -Be $Expected
}
}

Context 'Github Actions Warning Format' {
It "Header '<header>' and Message '<message>' returns '<expected>'" -TestCases @(
@{
Header = 'header'
Message = 'message'
Expected = @(
'::warning::header',
'::group::Message',
'message',
'::endgroup::'
)
}
@{
Header = 'header'
Message = @('message1', 'message2')
Expected = @(
'::warning::header',
'::group::Message',
'message1',
'message2',
'::endgroup::'
)
}
@{
Header = 'header'
Message = @(' message1', ' message2')
Expected = @(
'::warning::header',
'::group::Message',
'message1',
'message2',
'::endgroup::'
)
}
) {
Format-CIErrorMessage -CIFormat 'GithubActions' -CILogLevel 'Warning' -Header $Header -Message $Message | Should -Be $Expected
}
}
}
Expand Down

0 comments on commit 06ab106

Please sign in to comment.