Skip to content

Commit

Permalink
Add support for dumping native stacks with cdb (dotnet#82189)
Browse files Browse the repository at this point in the history 
Co-authored-by: Dan Moseley <[email protected]>
  • Loading branch information
@jkoritzinsky @danmoseley
jkoritzinsky and danmoseley authored Feb 24, 2023
1 parent 4ab53e1 commit eceb10c
Show file tree
Hide file tree
Showing 2 changed files with 79 additions and 22 deletions.
89 changes: 72 additions & 17 deletions src/tests/Common/Coreclr.TestWrapper/CoreclrTestWrapperLib.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -240,6 +240,8 @@ public class CoreclrTestWrapperLib
public const string COLLECT_DUMPS_ENVIRONMENT_VAR = "__CollectDumps";
public const string CRASH_DUMP_FOLDER_ENVIRONMENT_VAR = "__CrashDumpFolder";

public const string TEST_TARGET_ARCHITECTURE_ENVIRONMENT_VAR = "__TestArchitecture";

static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter outputWriter)
{
if (OperatingSystem.IsWindows())
Expand All @@ -254,11 +256,17 @@ static bool CollectCrashDump(Process process, string crashDumpPath, StreamWriter

static bool CollectCrashDumpWithMiniDumpWriteDump(Process process, string crashDumpPath, StreamWriter outputWriter)
{
bool collectedDump = false;
using (var crashDump = File.OpenWrite(crashDumpPath))
{
var flags = DbgHelp.MiniDumpType.MiniDumpWithFullMemory | DbgHelp.MiniDumpType.MiniDumpIgnoreInaccessibleMemory;
return DbgHelp.MiniDumpWriteDump(process.Handle, process.Id, crashDump.SafeFileHandle, flags, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
collectedDump = DbgHelp.MiniDumpWriteDump(process.Handle, process.Id, crashDump.SafeFileHandle, flags, IntPtr.Zero, IntPtr.Zero, IntPtr.Zero);
}
if (collectedDump)
{
TryPrintStackTraceFromDmp(crashDumpPath, outputWriter);
}
return collectedDump;
}

static bool CollectCrashDumpWithCreateDump(Process process, string crashDumpPath, StreamWriter outputWriter)
Expand Down Expand Up @@ -308,7 +316,7 @@ static bool CollectCrashDumpWithCreateDump(Process process, string crashDumpPath
private static string SKIP_LINE_TAG = "# <SKIP_LINE>";


static bool RunProcess(string fileName, string arguments)
static bool RunProcess(string fileName, string arguments, TextWriter outputWriter)
{
Process proc = new Process()
{
Expand All @@ -322,15 +330,15 @@ static bool RunProcess(string fileName, string arguments)
}
};

Console.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
outputWriter.WriteLine($"Invoking: {proc.StartInfo.FileName} {proc.StartInfo.Arguments}");
proc.Start();

Task<string> stdOut = proc.StandardOutput.ReadToEndAsync();
Task<string> stdErr = proc.StandardError.ReadToEndAsync();
if(!proc.WaitForExit(DEFAULT_TIMEOUT_MS))
{
proc.Kill(true);
Console.WriteLine($"Timedout: '{fileName} {arguments}");
outputWriter.WriteLine($"Timedout: '{fileName} {arguments}");
return false;
}

Expand All @@ -339,11 +347,11 @@ static bool RunProcess(string fileName, string arguments)
string error = stdErr.Result;
if (!string.IsNullOrWhiteSpace(output))
{
Console.WriteLine($"stdout: {output}");
outputWriter.WriteLine($"stdout: {output}");
}
if (!string.IsNullOrWhiteSpace(error))
{
Console.WriteLine($"stderr: {error}");
outputWriter.WriteLine($"stderr: {error}");
}
return true;
}
Expand All @@ -359,7 +367,7 @@ static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, Stream
{
if (OperatingSystem.IsLinux() || OperatingSystem.IsMacOS())
{
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}", Console.Out))
{
return false;
}
Expand All @@ -368,19 +376,19 @@ static bool TryPrintStackTraceFromCrashReport(string crashReportJsonFile, Stream
string userName = Environment.GetEnvironmentVariable("USER");
if (!string.IsNullOrEmpty(userName))
{
if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}"))
if (!RunProcess("sudo", $"chown {userName} {crashReportJsonFile}", Console.Out))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}"))
if (!RunProcess("sudo", $"ls -l {crashReportJsonFile}", Console.Out))
{
return false;
}

Console.WriteLine("=========================================");
if (!RunProcess("ls", $"-l {crashReportJsonFile}"))
if (!RunProcess("ls", $"-l {crashReportJsonFile}", Console.Out))
{
return false;
}
Expand Down Expand Up @@ -577,6 +585,38 @@ private static void AppendAddress(StringBuilder sb, string coreRoot, string nati
}
}

static bool TryPrintStackTraceFromDmp(string dmpFile, StreamWriter outputWriter)
{
string targetArchitecture = Environment.GetEnvironmentVariable(TEST_TARGET_ARCHITECTURE_ENVIRONMENT_VAR);
if (string.IsNullOrEmpty(targetArchitecture))
{
outputWriter.WriteLine($"Environment variable {TEST_TARGET_ARCHITECTURE_ENVIRONMENT_VAR} is not set.");
return false;
}

string cdbPath = $@"C:\Program Files (x86)\Windows Kits\10\Debuggers\{targetArchitecture}\cdb.exe";
if (!File.Exists(cdbPath))
{
outputWriter.WriteLine($"Unable to find cdb.exe at {cdbPath}");
return false;
}

var cdbScriptPath = Path.GetTempFileName();
// TODO: Add SOS support once we can easily download SOS to install.
File.WriteAllText(cdbScriptPath, """
~*k
q
""");

// cdb outputs the stacks directly, so we don't need to parse the output.
if (!RunProcess(cdbPath, $@"-c ""$<{cdbScriptPath}"" -z ""{dmpFile}""", outputWriter))
{
outputWriter.WriteLine("Unable to run cdb.exe");
return false;
}
return true;
}

// Finds all children processes starting with a process named childName
// The children are sorted in the order they should be dumped
static unsafe IEnumerable<Process> FindChildProcessesByName(Process process, string childName)
Expand Down Expand Up @@ -668,16 +708,16 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
MobileAppHandler.CheckExitCode(exitCode, testBinaryBase, category, outputWriter);
Task.WaitAll(copyOutput, copyError);

if (!OperatingSystem.IsWindows())
if (exitCode != 0)
{
// crashreport is only for non-windows.
if (exitCode != 0)
// Search for dump, if created.
if (Directory.Exists(crashDumpFolder))
{
// Search for dump, if created.
if (Directory.Exists(crashDumpFolder))
outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
// crashreport is only for non-windows.
if (!OperatingSystem.IsWindows())
{
outputWriter.WriteLine($"Test failed. Trying to see if dump file was created in {crashDumpFolder} since {startTime}");
DirectoryInfo crashDumpFolderInfo = new DirectoryInfo(crashDumpFolder);
var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.crashreport.json").OrderByDescending(f => f.CreationTime);
foreach (var dmpFile in dmpFilesInfo)
{
Expand All @@ -691,6 +731,21 @@ public int RunTest(string executable, string outputFile, string errorFile, strin
TryPrintStackTraceFromCrashReport(dmpFile.FullName, outputWriter);
}
}
else
{
var dmpFilesInfo = crashDumpFolderInfo.GetFiles("*.dmp").OrderByDescending(f => f.CreationTime);
foreach (var dmpFile in dmpFilesInfo)
{
if (dmpFile.CreationTime < startTime)
{
// No new files since test started.
outputWriter.WriteLine("Finished looking for *.dmp. No new files created.");
break;
}
outputWriter.WriteLine($"Processing {dmpFile.FullName}");
TryPrintStackTraceFromDmp(dmpFile.FullName, outputWriter);
}
}
}
}
}
Expand Down
12 changes: 7 additions & 5 deletions src/tests/Common/helixpublishwitharcade.proj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -569,7 +569,7 @@
DestinationFile="$(MergedPayloadsRootDirectory)\%(MergedPayloads.PayloadGroup).zip" />
</Target>


<PropertyGroup>
<EnableAzurePipelinesReporter>$(PublishTestResults)</EnableAzurePipelinesReporter>
<EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
Expand Down Expand Up @@ -628,6 +628,7 @@
<HelixPreCommand Include="set __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
<HelixPreCommand Include="set __CollectDumps=1" />
<HelixPreCommand Include="set __CrashDumpFolder=%HELIX_DUMP_FOLDER%" />
<HelixPreCommand Include="set __TestArchitecture=$(TargetArchitecture)" />
<HelixPreCommand Include="type %__TestEnv%" />
</ItemGroup>

Expand Down Expand Up @@ -677,6 +678,7 @@
<HelixPreCommand Include="export __TestTimeout=$(TimeoutPerTestInMilliseconds)" Condition=" '$(TimeoutPerTestInMilliseconds)' != '' " />
<HelixPreCommand Include="export __CollectDumps=1" />
<HelixPreCommand Include="export __CrashDumpFolder=$HELIX_DUMP_FOLDER" />
<HelixPreCommand Include="set __TestArchitecture=$(TargetArchitecture)" />
<HelixPreCommand Include="cat $__TestEnv" />

<HelixPostCommand Include="find $HELIX_WORKITEM_PAYLOAD -name '*testResults.xml' -exec cp {} $HELIX_DUMP_FOLDER\; " />
Expand Down Expand Up @@ -781,8 +783,8 @@
</ItemGroup>
</Target>

<Target Name="PrepareMergedStressHelixWorkItems"
BeforeTargets="PrepareMergedHelixWorkItems"
<Target Name="PrepareMergedStressHelixWorkItems"
BeforeTargets="PrepareMergedHelixWorkItems"
Condition="'$(StripeMergedStressWorkItems)'=='true'"
Outputs="%(_MergedStressWrapperMarker.Identity).dummy">
<PropertyGroup>
Expand Down Expand Up @@ -830,14 +832,14 @@
<CustomCommands>dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs)</CustomCommands>
<CustomCommands Condition=" '%(TestGroup)' != '' ">dotnet $(XUnitRunnerDll) %(XUnitWrapperDlls) $(XUnitRunnerArgs) -trait TestGroup=%(TestGroup)</CustomCommands>
</XHarnessApkToTest>

<XHarnessApkToTest Include="@(MergedPayloads->Metadata('PayloadGroup'))" Condition="'$(TargetsAndroid)' == 'true'">
<Arguments>--arg=env:TestExclusionListPath=TestExclusionList.txt</Arguments>
<AndroidPackageName>net.dot.%(PayloadGroup)</AndroidPackageName>
<AndroidInstrumentationName>net.dot.MonoRunner</AndroidInstrumentationName>
<TestTimeout Condition=" '$(TimeoutPerTestCollectionInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</TestTimeout>
</XHarnessApkToTest>

<XHarnessAppBundleToTest Include="@(LegacyPayloads->Metadata('PayloadZipFile'))" Condition="'$(TargetsAppleMobile)' == 'true'">
<TestTarget Condition="'$(TargetArchitecture)' == 'arm64'">ios-simulator-64</TestTarget>
<TestTarget Condition="'$(TargetArchitecture)' == 'x64'">ios-simulator-64</TestTarget>
Expand Down

0 comments on commit eceb10c

Please sign in to comment.