Skip to content

Commit

Permalink
Improve crash dump debugging in CI (microsoft#2454)
Browse files Browse the repository at this point in the history
  • Loading branch information
@thhous-msft
thhous-msft authored Mar 2, 2022
1 parent b72c6b3 commit d381554
Show file tree
Hide file tree
Showing 8 changed files with 197 additions and 18 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ else() #!WIN32

if(QUIC_ENABLE_SANITIZERS)
message(STATUS "Configuring sanitizers")
list(APPEND QUIC_COMMON_FLAGS -fsanitize=address,leak,undefined -fsanitize-address-use-after-scope -Og -fno-omit-frame-pointer -fno-optimize-sibling-calls)
list(APPEND QUIC_COMMON_FLAGS -fsanitize=address,leak,undefined,alignment -fsanitize-address-use-after-scope -Og -fno-omit-frame-pointer -fno-optimize-sibling-calls)
if (CX_PLATFORM STREQUAL "darwin")
list(APPEND QUIC_COMMON_FLAGS -gdwarf)
else()
Expand Down
26 changes: 17 additions & 9 deletions scripts/log.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,11 +196,15 @@ function Log-Stop {
Write-Debug "Decoding LTTng into BabelTrace format ($BableTraceFile)"
babeltrace --names all $TempDir/* > $BableTraceFile

Write-Host "Decoding into human-readable text: $ClogOutputDecodeFile"
$Command = "$Clog2Text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile --showTimestamp --showCpuInfo"
Write-Host $Command
Invoke-Expression $Command | Write-Debug
Remove-Item -Path $BableTraceFile -Force | Out-Null
Write-Host "Babeltrace ran. Compile clog2text and run the following command"
$Command = "clog2text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile --showTimestamp --showCpuInfo"
$Command

# Write-Host "Decoding into human-readable text: $ClogOutputDecodeFile"
# $Command = "$Clog2Text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile --showTimestamp --showCpuInfo"
# Write-Host $Command
# Invoke-Expression $Command | Write-Debug
# Remove-Item -Path $BableTraceFile -Force | Out-Null
}

Invoke-Expression "lttng destroy $InstanceName" | Write-Debug
Expand Down Expand Up @@ -234,10 +238,14 @@ function Log-Decode {
Write-Host "Decoding LTTng into BabelTrace format ($BableTraceFile)"
babeltrace --names all $DecompressedLogs/* > $BableTraceFile

Write-Host "Decoding Babeltrace into human text using CLOG"
$Command = "$Clog2Text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile"
Write-Host $Command
Invoke-Expression $Command
Write-Host "Babeltrace ran. Compile clog2text and run the following command"
$Command = "clog2text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile"
$Command

#Write-Host "Decoding Babeltrace into human text using CLOG"
#$Command = "$Clog2Text_lttng -i $BableTraceFile -s $SideCar -o $ClogOutputDecodeFile"
#Write-Host $Command
#Invoke-Expression $Command
}
}

Expand Down
3 changes: 2 additions & 1 deletion scripts/prepare-machine.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,7 @@ if ($IsWindows) {
sudo apt-add-repository ppa:lttng/stable-2.12
sudo apt-get update
sudo apt-get install -y lttng-tools
sudo apt-get install -y gdb

# Enable core dumps for the system.
Write-Host "[$(Get-Date)] Setting core dump size limit..."
Expand Down Expand Up @@ -401,7 +402,7 @@ if ($IsWindows) {
} elseif ($IsMacOS) {
if ($Configuration -eq "Test") {
Write-Host "[$(Get-Date)] Setting core dump pattern..."
sudo sysctl -w kern.corefile=%N.%P.%H.core
sudo sysctl -w kern.corefile=%N.%P.core
}
}

Expand Down
87 changes: 84 additions & 3 deletions scripts/run-executable.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ param (
[switch]$CodeCoverage = $false,

[Parameter(Mandatory = $false)]
[switch]$AZP = $false
[switch]$AZP = $false,

[Parameter(Mandatory = $false)]
[string]$ExtraArtifactDir = ""
)

Set-StrictMode -Version 'Latest'
Expand Down Expand Up @@ -139,8 +142,13 @@ $LogScript = Join-Path $RootDir "scripts" "log.ps1"
$ExeName = Split-Path $Path -Leaf
$CoverageName = "$(Split-Path $Path -LeafBase).cov"

$ExeLogFolder = $ExeName
if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
$ExeLogFolder += "_$ExtraArtifactDir"
}

# Path for log files.
$LogDir = Join-Path $RootDir "artifacts" "logs" $ExeName (Get-Date -UFormat "%m.%d.%Y.%T").Replace(':','.')
$LogDir = Join-Path $RootDir "artifacts" "logs" $ExeLogFolder (Get-Date -UFormat "%m.%d.%Y.%T").Replace(':','.')
New-Item -Path $LogDir -ItemType Directory -Force | Out-Null

# Folder for coverage files
Expand Down Expand Up @@ -221,7 +229,7 @@ function Start-Executable {
}
} else {
$pinfo.FileName = "bash"
$pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 $($Path) $($Arguments) && echo Done`""
$pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $($Path) $($Arguments) && echo Done`""
$pinfo.WorkingDirectory = $LogDir
}
}
Expand Down Expand Up @@ -253,11 +261,72 @@ function PrintDumpCallStack($DumpFile) {
Write-Host " $(Split-Path $DumpFile -Leaf)"
Write-Host "=================================================================================="
$Output -replace "quit:", "=================================================================================="
$Output | Out-File "$DumpFile.txt"
} catch {
# Silently fail
}
}

function PrintLldbCoreCallStack($CoreFile) {
try {
$Output = lldb $Path -c $CoreFile -b -o "`"bt all`""
Write-Host "=================================================================================="
Write-Host " $(Split-Path $CoreFile -Leaf)"
Write-Host "=================================================================================="
# Find line containing Current thread
$Found = $false
$LastThreadStart = 0
for ($i = 0; $i -lt $Output.Length; $i++) {
if ($Output[$i] -like "*stop reason =*") {
if ($Found) {
break
}
$LastThreadStart = $i
}
if ($Output[$i] -like "*quic_bugcheck*") {
$Found = $true
for ($j = $LastThreadStart; $j -lt $i; $j++) {
$Output[$j]
}
}
if ($Found) {
$Output[$i]
}
}
if (!$Found) {
$Output | Join-String -Separator "`n"
}
$Output | Join-String -Separator "`n" | Out-File "$CoreFile.txt"
} catch {
# Silently Fail
}
}

function PrintGdbCoreCallStack($CoreFile) {
try {
$Output = gdb $Path $CoreFile -batch -ex "`"bt`"" -ex "`"quit`""
Write-Host "=================================================================================="
Write-Host " $(Split-Path $CoreFile -Leaf)"
Write-Host "=================================================================================="
# Find line containing Current thread
$Found = $false
for ($i = 0; $i -lt $Output.Length; $i++) {
if ($Output[$i] -like "*Current thread*") {
$Found = $true
}
if ($Found) {
$Output[$i]
}
}
if (!$Found) {
$Output | Join-String -Separator "`n"
}
$Output | Join-String -Separator "`n" | Out-File "$CoreFile.txt"
} catch {
# Silently Fail
}
}

# Waits for the executable to finish and processes the results.
function Wait-Executable($Exe) {
$stdout = $null
Expand Down Expand Up @@ -307,6 +376,18 @@ function Wait-Executable($Exe) {
}
$KeepOutput = $true
}
$CoreFiles = (Get-ChildItem $LogDir) | Where-Object { $_.Extension -eq ".core" }
if ($CoreFiles) {
LogWrn "Core file(s) generated"
foreach ($File in $CoreFiles) {
if ($IsMacOS) {
PrintLldbCoreCallStack $File
} else {
PrintGdbCoreCallStack $File
}
}
$KeepOutput = $true
}
} catch {
LogWrn $_
LogErr "Treating exception as failure!"
Expand Down
87 changes: 84 additions & 3 deletions scripts/run-gtest.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,10 @@ param (
[switch]$AZP = $false,

[Parameter(Mandatory = $false)]
[switch]$ErrorsAsWarnings = $false
[switch]$ErrorsAsWarnings = $false,

[Parameter(Mandatory = $false)]
[string]$ExtraArtifactDir = ""
)

Set-StrictMode -Version 'Latest'
Expand Down Expand Up @@ -187,8 +190,13 @@ $LogScript = Join-Path $RootDir "scripts" "log.ps1"
$TestExeName = Split-Path $Path -Leaf
$CoverageName = "$(Split-Path $Path -LeafBase).cov"

$ExeLogFolder = $TestExeName
if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
$ExeLogFolder += "_$ExtraArtifactDir"
}

# Folder for log files.
$LogDir = Join-Path $RootDir "artifacts" "logs" $TestExeName (Get-Date -UFormat "%m.%d.%Y.%T").Replace(':','.')
$LogDir = Join-Path $RootDir "artifacts" "logs" $ExeLogFolder (Get-Date -UFormat "%m.%d.%Y.%T").Replace(':','.')
New-Item -Path $LogDir -ItemType Directory -Force | Out-Null

# Folder for coverage files
Expand Down Expand Up @@ -315,7 +323,7 @@ function Start-TestExecutable([String]$Arguments, [String]$OutputDir) {
}
} else {
$pinfo.FileName = "bash"
$pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 $($Path) $($Arguments) && echo Done`""
$pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $($Path) $($Arguments) && echo Done`""
$pinfo.WorkingDirectory = $OutputDir
}
}
Expand Down Expand Up @@ -415,11 +423,72 @@ function PrintDumpCallStack($DumpFile) {
Write-Host " $(Split-Path $DumpFile -Leaf)"
Write-Host "=================================================================================="
$Output -replace "quit:", "=================================================================================="
$Output | Out-File "$DumpFile.txt"
} catch {
# Silently fail
}
}

function PrintLldbCoreCallStack($CoreFile) {
try {
$Output = lldb $Path -c $CoreFile -b -o "`"bt all`""
Write-Host "=================================================================================="
Write-Host " $(Split-Path $CoreFile -Leaf)"
Write-Host "=================================================================================="
# Find line containing Current thread
$Found = $false
$LastThreadStart = 0
for ($i = 0; $i -lt $Output.Length; $i++) {
if ($Output[$i] -like "*stop reason =*") {
if ($Found) {
break
}
$LastThreadStart = $i
}
if ($Output[$i] -like "*quic_bugcheck*") {
$Found = $true
for ($j = $LastThreadStart; $j -lt $i; $j++) {
$Output[$j]
}
}
if ($Found) {
$Output[$i]
}
}
if (!$Found) {
$Output | Join-String -Separator "`n"
}
$Output | Join-String -Separator "`n" | Out-File "$CoreFile.txt"
} catch {
# Silently Fail
}
}

function PrintGdbCoreCallStack($CoreFile) {
try {
$Output = gdb $Path $CoreFile -batch -ex "`"bt`"" -ex "`"quit`""
Write-Host "=================================================================================="
Write-Host " $(Split-Path $CoreFile -Leaf)"
Write-Host "=================================================================================="
# Find line containing Current thread
$Found = $false
for ($i = 0; $i -lt $Output.Length; $i++) {
if ($Output[$i] -like "*Current thread*") {
$Found = $true
}
if ($Found) {
$Output[$i]
}
}
if (!$Found) {
$Output | Join-String -Separator "`n"
}
$Output | Join-String -Separator "`n" | Out-File "$CoreFile.txt"
} catch {
# Silently Fail
}
}

# Waits for the executable to finish and processes the results.
function Wait-TestCase($TestCase) {
$ProcessCrashed = $false
Expand Down Expand Up @@ -463,6 +532,18 @@ function Wait-TestCase($TestCase) {
}
$ProcessCrashed = $true
}
$CoreFiles = (Get-ChildItem $TestCase.LogDir) | Where-Object { $_.Extension -eq ".core" }
if ($CoreFiles) {
LogWrn "Core file(s) generated"
foreach ($File in $CoreFiles) {
if ($IsMacOS) {
PrintLldbCoreCallStack $File
} else {
PrintGdbCoreCallStack $File
}
}
$ProcessCrashed = $true
}
} catch {
LogWrn "Treating exception as crash!"
$ProcessCrashed = $true
Expand Down
4 changes: 4 additions & 0 deletions scripts/spin.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -168,5 +168,9 @@ if ($AZP) {
$Arguments += " -AZP"
}

if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
$Arguments += " -ExtraArtifactDir $ExtraArtifactDir"
}

# Run the script.
Invoke-Expression ($RunExecutable + " " + $Arguments)
4 changes: 4 additions & 0 deletions scripts/test.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,6 +305,10 @@ if ($ErrorsAsWarnings) {
$TestArguments += " -ErrorsAsWarnings"
}

if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
$TestArguments += " -ExtraArtifactDir $ExtraArtifactDir"
}

# Run the script.
if (!$Kernel -and !$SkipUnitTests) {
Invoke-Expression ($RunTest + " -Path $MsQuicCoreTest " + $TestArguments)
Expand Down
2 changes: 1 addition & 1 deletion src/inc/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ if (HAS_GUARDCF)
endif()

if (NOT MSVC AND QUIC_ENABLE_SANITIZERS)
target_link_libraries(base_link INTERFACE -fsanitize=address,leak,undefined)
target_link_libraries(base_link INTERFACE -fsanitize=address,leak,undefined,alignment)
endif()

if (NOT MSVC AND NOT APPLE AND NOT ANDROID)
Expand Down

0 comments on commit d381554

Please sign in to comment.