Add JUnit XML output (pester#1748)

Add JUnitXml output option to output JUnit xml result Co-authored-by: Frode Flaten <[email protected]>
helenachi-tsc · Nov 1, 2020 · ca78cfc · ca78cfc
1 parent 3c1b601
commit ca78cfc
Show file tree

Hide file tree

Showing 8 changed files with 459 additions and 254 deletions.
diff --git a/src/Pester.RSpec.ps1 b/src/Pester.RSpec.ps1
@@ -373,10 +373,10 @@ function New-PesterContainer {
     [CmdletBinding(DefaultParameterSetName="Path")]
     param(
         [Parameter(Mandatory, ParameterSetName = "Path")]
-        [String] $Path,
+        [String[]] $Path,
 
         [Parameter(Mandatory, ParameterSetName = "ScriptBlock")]
-        [ScriptBlock] $ScriptBlock,
+        [ScriptBlock[]] $ScriptBlock,
 
         [Collections.IDictionary[]] $Data
     )
@@ -390,18 +390,22 @@ function New-PesterContainer {
         # the @() is significant here, it will make it iterate even if there are no data
         # which allows scriptblocks without data to run
         foreach ($d in @($dt)) {
-            New-BlockContainerObject -ScriptBlock $ScriptBlock -Data $d
+            foreach ($sb in $ScriptBlock) {
+                New-BlockContainerObject -ScriptBlock $sb -Data $d
+            }
         }
     }
 
     if ("Path" -eq $kind) {
         # the @() is significant here, it will make it iterate even if there are no data
         # which allows files without data to run
         foreach ($d in @($dt)) {
-            # resolve the path we are given in the same way we would resolve -Path
-            $files = @(Find-File -Path $Path -ExcludePath $PesterPreference.Run.ExcludePath.Value -Extension $PesterPreference.Run.TestExtension.Value)
-            foreach ($file in $files) {
-                New-BlockContainerObject -File $file -Data $d
+            foreach ($p in $Path) {
+                # resolve the path we are given in the same way we would resolve -Path on Invoke-Pester
+                $files = @(Find-File -Path $p -ExcludePath $PesterPreference.Run.ExcludePath.Value -Extension $PesterPreference.Run.TestExtension.Value)
+                foreach ($file in $files) {
+                    New-BlockContainerObject -File $file -Data $d
+                }
             }
         }
     }

diff --git a/src/Pester.ps1 b/src/Pester.ps1
@@ -406,7 +406,8 @@ function Invoke-Pester {
     [PesterConfiguration]::Default.TestResult
     ----------
     TestResult.Enabled - Enable TestResult.
-    TestResult.OutputFormat - Format to use for test result report. Possible values: NUnit2.5
+    TestResult.OutputFormat - Format to use for test result report. Possible values:  NUnitXml, JUnitXml
+        Default is: NUnitXml
     TestResult.OutputPath - Path relative to the current directory where test result report is saved.
         Default is: testResults.xml
     TestResult.OutputEncoding - Encoding of the output file. Currently UTF8
@@ -465,8 +466,7 @@ function Invoke-Pester {
     .PARAMETER OutputFormat
     (Deprecated v4)
     Replace with ConfigurationProperty TestResult.OutputFormat
-    The format of output. Currently NUnitXml is supported.
-    Note that JUnitXml is not currently supported in Pester 5.
+    The format of output. Currently NUnitXml and JUnitXml is supported.
 
     .PARAMETER PassThru
     Replace with ConfigurationProperty Run.PassThru
@@ -864,10 +864,6 @@ function Invoke-Pester {
 
                 if ($PSBoundParameters.ContainsKey('OutputFormat')) {
                     if ($null -ne $OutputFormat -and 0 -lt @($OutputFormat).Count) {
-                        if ("JUnitXml" -eq $OutputFormat) {
-                            throw "JUnitXml is currently not supported in Pester 5."
-                        }
-
                         $Configuration.TestResult.OutputFormat = $OutputFormat
                     }
 
@@ -1091,7 +1087,7 @@ function Invoke-Pester {
             }
 
             if ($PesterPreference.TestResult.Enabled.Value) {
-                Export-NunitReport $run $PesterPreference.TestResult.OutputPath.Value
+                Export-PesterResults -Result $run -Path $PesterPreference.TestResult.OutputPath.Value -Format $PesterPreference.TestResult.OutputFormat.Value
             }
 
             if ($PesterPreference.CodeCoverage.Enabled.Value) {

diff --git a/src/Pester.psd1 b/src/Pester.psd1
@@ -54,8 +54,8 @@
         # export
         'Export-NUnitReport'
         'ConvertTo-NUnitReport'
-        # 'Export-JUnitReport'
-        # 'ConvertTo-JUnitReport'
+        'Export-JUnitReport'
+        'ConvertTo-JUnitReport'
         'ConvertTo-Pester4Result'
 
         # config

diff --git a/src/Pester.psm1 b/src/Pester.psm1
@@ -40,8 +40,8 @@ $script:SafeCommands['Set-DynamicParameterVariable'] = $ExecutionContext.Session
     # export
     'Export-NunitReport'
     'ConvertTo-NUnitReport'
-    # 'Export-JUnitReport' does not work yet, it needs similar rework as NUnit to work with the new structure
-    # 'ConvertTo-JUnitReport'
+    'Export-JUnitReport'
+    'ConvertTo-JUnitReport'
     'ConvertTo-Pester4Result'
 
     # legacy

diff --git a/src/csharp/Pester/Configuration.cs b/src/csharp/Pester/Configuration.cs
@@ -718,7 +718,7 @@ public static TestResultConfiguration ShallowClone(TestResultConfiguration confi
         public TestResultConfiguration() : base("TestResult configuration.")
         {
             Enabled = new BoolOption("Enable TestResult.", false);
-            OutputFormat = new StringOption("Format to use for test result report. Possible values: NUnit2.5", "NUnit2.5");
+            OutputFormat = new StringOption("Format to use for test result report. Possible values: NUnitXml, JUnitXml", "NUnitXml");
             OutputPath = new StringOption("Path relative to the current directory where test result report is saved.", "testResults.xml");
             OutputEncoding = new StringOption("Encoding of the output file.", "UTF8");
             TestSuiteName = new StringOption("Set the name assigned to the root 'test-suite' element.", "Pester");

diff --git a/src/functions/TestResults.ps1 b/src/functions/TestResults.ps1
@@ -58,6 +58,18 @@ function Export-NUnitReport {
     Export-XmlReport -Result $Result -Path $Path -Format NUnitXml
 }
 
+function Export-JUnitReport {
+    param (
+        [parameter(Mandatory = $true, ValueFromPipeline = $true)]
+        $Result,
+
+        [parameter(Mandatory = $true)]
+        [String] $Path
+    )
+
+    Export-XmlReport -Result $Result -Path $Path -Format JUnitXml
+}
+
 function Export-XmlReport {
     param (
         [parameter(Mandatory = $true, ValueFromPipeline = $true)]
@@ -254,7 +266,7 @@ function Write-NUnitTestSuiteElements($Node, [System.Xml.XmlWriter] $XmlWriter,
             # skip blocks that were discovered but did not run
             continue
         }
-        Write-NUnitTestSuiteElements -Node $action -XmlWriter $XmlWriter -Path ($action.Path -join '.')
+        Write-NUnitTestSuiteElements -Node $action -XmlWriter $XmlWriter -Path $action.ExpandedPath
     }
 
     $suites = @(
@@ -307,7 +319,12 @@ function Write-JUnitReport($Result, [System.Xml.XmlWriter] $XmlWriter) {
 
     $testSuiteNumber = 0
     foreach ($container in $Result.Containers) {
-        Write-JUnitTestSuiteElements -XmlWriter $XmlWriter -Node $container -Id $testSuiteNumber
+        if (-not $container.ShouldRun) {
+            # skip containers that were discovered but none of their tests run
+            continue
+        }
+
+        Write-JUnitTestSuiteElements -XmlWriter $XmlWriter -Container $container -Id $testSuiteNumber
         $testSuiteNumber++
     }
 
@@ -352,41 +369,33 @@ function Write-JUnitTestResultAttributes($Result, [System.Xml.XmlWriter] $XmlWri
     $XmlWriter.WriteAttributeString('xmlns', 'xsi', $null, 'http://www.w3.org/2001/XMLSchema-instance')
     $XmlWriter.WriteAttributeString('xsi', 'noNamespaceSchemaLocation', [Xml.Schema.XmlSchema]::InstanceNamespace , 'junit_schema_4.xsd')
     $XmlWriter.WriteAttributeString('name', $Result.Configuration.TestResult.TestSuiteName.Value)
-    $XmlWriter.WriteAttributeString('tests', $Result.PassedCount)
-    $XmlWriter.WriteAttributeString('errors', '0')
+    $XmlWriter.WriteAttributeString('tests', $Result.TotalCount)
+    $XmlWriter.WriteAttributeString('errors', $Result.FailedContainersCount + $Result.FailedBlocksCount)
     $XmlWriter.WriteAttributeString('failures', $Result.FailedCount)
     $XmlWriter.WriteAttributeString('disabled', $Result.NotRunCount + $Result.SkippedCount)
     $XmlWriter.WriteAttributeString('time', ($Result.Duration.TotalSeconds.ToString('0.000', [System.Globalization.CultureInfo]::InvariantCulture)))
 }
 
-function Write-JUnitTestSuiteElements($Node, [System.Xml.XmlWriter] $XmlWriter, [uint16] $Id) {
+function Write-JUnitTestSuiteElements($Container, [System.Xml.XmlWriter] $XmlWriter, [uint16] $Id) {
     $XmlWriter.WriteStartElement('testsuite')
 
-    Write-JUnitTestSuiteAttributes -Action $Node -XmlWriter $XmlWriter -Package $Node.Name -Id $Id
-
-    $testCases = foreach ($al1 in $node.Actions) {
-        if ($al1.Type -ne 'TestCase') {
-            foreach ($al2 in $al1.Actions) {
-                if ($al2.Type -ne 'TestCase') {
-                    foreach ($alt3 in $al2.Actions) {
-                        $path = "$($al1.Name).$($al2.Name).$($alt3.Name)"
-                        $alt3 | & $SafeCommands['Add-Member'] -PassThru -MemberType NoteProperty -Name Path -Value $path
-                    }
-                }
-                else {
-                    $path = "$($al1.Name).$($al2.Name)"
-                    $al2 | & $SafeCommands['Add-Member'] -PassThru -MemberType NoteProperty -Name Path -Value $path
-                }
-            }
-        }
-        else {
-            $path = "$($al1.Name)"
-            $al1 | & $SafeCommands['Add-Member'] -PassThru -MemberType NoteProperty -Name Path -Value $path
-        }
+    if ("File" -eq $Container.Type) {
+        $path = $Container.Item.FullName
+    }
+    elseif ("ScriptBlock" -eq $Container.Type) {
+        $path = "<ScriptBlock>$($Container.Item.File):$($Container.Item.StartPosition.StartLine)"
+    }
+    else {
+        throw "Container type '$($Container.Type)' is not supported."
     }
 
-    foreach ($t in $testCases) {
-        Write-JUnitTestCaseElements -Action $t -XmlWriter $XmlWriter -Package $Node.Name
+    Write-JUnitTestSuiteAttributes -Action $Container -XmlWriter $XmlWriter -Package $path -Id $Id
+
+
+    $testResults = [Pester.Factory]::CreateCollection()
+    Fold-Container -Container $Container -OnTest { param ($t) if ($t.ShouldRun) { $testResults.Add($t) } }
+    foreach ($t in $testResults) {
+        Write-JUnitTestCaseElements -TestResult $t -XmlWriter $XmlWriter -Package $path
     }
 
     $XmlWriter.WriteEndElement()
@@ -395,7 +404,7 @@ function Write-JUnitTestSuiteElements($Node, [System.Xml.XmlWriter] $XmlWriter,
 function Write-JUnitTestSuiteAttributes($Action, [System.Xml.XmlWriter] $XmlWriter, [string] $Package, [uint16] $Id) {
     $environment = Get-RunTimeEnvironment
 
-    $XmlWriter.WriteAttributeString('name', $Action.Name)
+    $XmlWriter.WriteAttributeString('name', $Package)
     $XmlWriter.WriteAttributeString('tests', $Action.TotalCount)
     $XmlWriter.WriteAttributeString('errors', '0')
     $XmlWriter.WriteAttributeString('failures', $Action.FailedCount)
@@ -422,18 +431,18 @@ function Write-JUnitTestSuiteAttributes($Action, [System.Xml.XmlWriter] $XmlWrit
     $XmlWriter.WriteEndElement()
 }
 
-function Write-JUnitTestCaseElements($Action, [System.Xml.XmlWriter] $XmlWriter, [string] $Package) {
+function Write-JUnitTestCaseElements($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $Package) {
     $XmlWriter.WriteStartElement('testcase')
 
-    Write-JUnitTestCaseAttributes -Action $Action -XmlWriter $XmlWriter -ClassName $Package
+    Write-JUnitTestCaseAttributes -TestResult $TestResult -XmlWriter $XmlWriter -ClassName $Package
 
     $XmlWriter.WriteEndElement()
 }
 
-function Write-JUnitTestCaseAttributes($Action, [System.Xml.XmlWriter] $XmlWriter, [string] $ClassName) {
-    $XmlWriter.WriteAttributeString('name', $Action.Path)
+function Write-JUnitTestCaseAttributes($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $ClassName) {
+    $XmlWriter.WriteAttributeString('name', $TestResult.ExpandedPath)
 
-    $statusElementName = switch ($Action.Result) {
+    $statusElementName = switch ($TestResult.Result) {
         Passed {
             $null
         }
@@ -447,20 +456,22 @@ function Write-JUnitTestCaseAttributes($Action, [System.Xml.XmlWriter] $XmlWrite
         }
     }
 
-    $XmlWriter.WriteAttributeString('status', $Action.Result)
+    $XmlWriter.WriteAttributeString('status', $TestResult.Result)
     $XmlWriter.WriteAttributeString('classname', $ClassName)
     $XmlWriter.WriteAttributeString('assertions', '0')
-    $XmlWriter.WriteAttributeString('time', $Action.Duration.TotalSeconds.ToString('0.000', [System.Globalization.CultureInfo]::InvariantCulture))
+    $XmlWriter.WriteAttributeString('time', $TestResult.Duration.TotalSeconds.ToString('0.000', [System.Globalization.CultureInfo]::InvariantCulture))
 
     if ($null -ne $statusElementName) {
-        Write-JUnitTestCaseMessageElements -Action $Action -XmlWriter $XmlWriter -StatusElementName $statusElementName
+        Write-JUnitTestCaseMessageElements -TestResult $TestResult -XmlWriter $XmlWriter -StatusElementName $statusElementName
     }
 }
 
-function Write-JUnitTestCaseMessageElements($Action, [System.Xml.XmlWriter] $XmlWriter, [string] $StatusElementName) {
+function Write-JUnitTestCaseMessageElements($TestResult, [System.Xml.XmlWriter] $XmlWriter, [string] $StatusElementName) {
     $XmlWriter.WriteStartElement($StatusElementName)
 
-    $XmlWriter.WriteAttributeString('message', $Action.FailureMessage) #TODO: Add stacktrace
+    $result = Get-ErrorForXmlReport -TestResult $TestResult
+    $XmlWriter.WriteAttributeString('message', $result.FailureMessage)
+    $XmlWriter.WriteString($result.StackTrace)
 
     $XmlWriter.WriteEndElement()
 }
@@ -699,45 +710,54 @@ function Write-NUnitTestCaseAttributes($TestResult, [System.Xml.XmlWriter] $XmlW
             # TODO: remove monkey patching the error message when parent setup failed so this test never run
             # TODO: do not format the errors here, instead format them in the core using some unified function so we get the same thing on the screen and in nunit
 
-            $failureMessage = if (($TestResult.ShouldRun -and -not $TestResult.Executed)) {
-                "This test should run but it did not. Most likely a setup in some parent block failed."
-            }
-            else {
-                $multipleErrors = 1 -lt $TestResult.ErrorRecord.Count
-
-                if ($multipleErrors) {
-                    $c = 0
-                    $(foreach ($err in $TestResult.ErrorRecord) {
-                        "[$(($c++))] $($err.DisplayErrorMessage)"
-                    }) -join [Environment]::NewLine
-                }
-                else {
-                    $TestResult.ErrorRecord.DisplayErrorMessage
-                }
-            }
-
-            $stackTrace = & {
-                $multipleErrors = 1 -lt $TestResult.ErrorRecord.Count
-
-                if ($multipleErrors) {
-                    $c = 0
-                    $(foreach ($err in $TestResult.ErrorRecord) {
-                        "[$(($c++))] $($err.DisplayStackTrace)"
-                    }) -join [Environment]::NewLine
-                }
-                else {
-                    [string] $TestResult.ErrorRecord.DisplayStackTrace
-                }
-            }
+            $result = Get-ErrorForXmlReport -TestResult $TestResult
 
-            $xmlWriter.WriteElementString('message', $failureMessage)
-            $XmlWriter.WriteElementString('stack-trace', $stackTrace)
+            $xmlWriter.WriteElementString('message', $result.FailureMessage)
+            $XmlWriter.WriteElementString('stack-trace', $result.StackTrace)
             $XmlWriter.WriteEndElement() # Close failure tag
             break
         }
     }
 }
 
+function Get-ErrorForXmlReport ($TestResult) {
+    $failureMessage = if (($TestResult.ShouldRun -and -not $TestResult.Executed)) {
+        "This test should run but it did not. Most likely a setup in some parent block failed."
+    }
+    else {
+        $multipleErrors = 1 -lt $TestResult.ErrorRecord.Count
+
+        if ($multipleErrors) {
+            $c = 0
+            $(foreach ($err in $TestResult.ErrorRecord) {
+                "[$(($c++))] $($err.DisplayErrorMessage)"
+            }) -join [Environment]::NewLine
+        }
+        else {
+            $TestResult.ErrorRecord.DisplayErrorMessage
+        }
+    }
+
+    $st = & {
+        $multipleErrors = 1 -lt $TestResult.ErrorRecord.Count
+
+        if ($multipleErrors) {
+            $c = 0
+            $(foreach ($err in $TestResult.ErrorRecord) {
+                "[$(($c++))] $($err.DisplayStackTrace)"
+            }) -join [Environment]::NewLine
+        }
+        else {
+            [string] $TestResult.ErrorRecord.DisplayStackTrace
+        }
+    }
+
+    @{
+        FailureMessage = $failureMessage
+        StackTrace = $st
+    }
+}
+
 function Get-RunTimeEnvironment() {
     # based on what we found during startup, use the appropriate cmdlet
     $computerName = $env:ComputerName

diff --git a/tst/Pester.RSpec.Configuration.ts.ps1 b/tst/Pester.RSpec.Configuration.ts.ps1
@@ -119,8 +119,8 @@ i -PassThru:$PassThru {
             [PesterConfiguration]::Default.TestResult.Enabled.Value | Verify-False
         }
 
-        t "TestResult.OutputFormat is NUnit2.5" {
-            [PesterConfiguration]::Default.TestResult.OutputFormat.Value | Verify-Equal "NUnit2.5"
+        t "TestResult.OutputFormat is NUnitXml" {
+            [PesterConfiguration]::Default.TestResult.OutputFormat.Value | Verify-Equal "NUnitXml"
         }
 
         t "TestResult.OutputPath is testResults.xml" {