Skip to content

Commit

Permalink
Fix SuperIlc to work on Linux / OSX in composite mode (dotnet#33358)
Browse files Browse the repository at this point in the history 
1) I have generalized the concept of framework exclusions so that
the CPAOT runner can consume it - without it we were picking up
"CommandLine.dll" in the Core_Root folder as a Crossgen2 compilation
input and R2RDump was subsequently failing to dump the file as
CommandLine.dll is one of the stupid assemblies where the assembly
name differs from the PE file name (the assembly name is
'commandline' causing a mismatch on case-sensitive file systems).

2) On Linux / OSX, we need to copy around a bunch of native libraries
to the output, e.g. libSystem.Globalization.Native.so.

Thanks

Tomas
  • Loading branch information
@trylek
trylek authored Mar 11, 2020
1 parent 458b95b commit 17b28ba
Show file tree
Hide file tree
Showing 5 changed files with 97 additions and 61 deletions.
10 changes: 9 additions & 1 deletion src/coreclr/src/tools/ReadyToRun.SuperIlc/BuildFolder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public BuildFolder(
public static BuildFolder FromDirectory(string inputDirectory, IEnumerable<CompilerRunner> compilerRunners, string outputRoot, BuildOptions options)
{
List<string> compilationInputFiles = new List<string>();
List<string> passThroughFiles = new List<string>();
HashSet<string> passThroughFiles = new HashSet<string>();
List<string> mainExecutables = new List<string>();
List<string> executionScripts = new List<string>();

Expand Down Expand Up @@ -173,6 +173,14 @@ public static BuildFolder FromDirectory(string inputDirectory, IEnumerable<Compi
passThroughFiles.Add(Path.Combine(options.CoreRootDirectory.FullName, lib.AppendOSDllSuffix()));
}
}
else
{
// Several native lib*.so / dylib are needed by the runtime
foreach (string nativeLib in Directory.EnumerateFiles(options.CoreRootDirectory.FullName, "lib*".AppendOSDllSuffix()))
{
passThroughFiles.Add(nativeLib);
}
}
}

foreach (CompilerRunner runner in compilerRunners)
Expand Down
43 changes: 9 additions & 34 deletions src/coreclr/src/tools/ReadyToRun.SuperIlc/BuildFolderSet.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,33 +16,6 @@ namespace ReadyToRun.SuperIlc
{
public class BuildFolderSet
{
class FrameworkExclusion
{
public readonly string SimpleName;
public readonly string Reason;
public readonly bool Crossgen2Only;

public FrameworkExclusion(string simpleName, string reason, bool crossgen2Only = false)
{
SimpleName = simpleName;
Reason = reason;
Crossgen2Only = crossgen2Only;
}
}

private static readonly FrameworkExclusion[] s_frameworkExclusions =
{
new FrameworkExclusion("CommandLine", "Not a framework assembly"),
new FrameworkExclusion("R2RDump", "Not a framework assembly"),
new FrameworkExclusion("System.Runtime.WindowsRuntime", "WinRT is currently not supported"),
new FrameworkExclusion("xunit.performance.api", "Not a framework assembly"),

// TODO (DavidWr): IBC-related failures
new FrameworkExclusion("Microsoft.CodeAnalysis.CSharp", "Ibc TypeToken 6200019a has type token which resolves to a nil token", crossgen2Only: true),
new FrameworkExclusion("Microsoft.CodeAnalysis", "Ibc TypeToken 620001af unable to find external typedef", crossgen2Only: true),
new FrameworkExclusion("Microsoft.CodeAnalysis.VisualBasic", "Ibc TypeToken 620002ce unable to find external typedef", crossgen2Only: true),
};

private readonly IEnumerable<BuildFolder> _buildFolders;

private readonly IEnumerable<CompilerRunner> _compilerRunners;
Expand Down Expand Up @@ -274,10 +247,9 @@ public bool CompileFramework()
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_options.CoreRootDirectory.FullName))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
FrameworkExclusion exclusion = s_frameworkExclusions.FirstOrDefault(asm => asm.SimpleName.Equals(simpleName, StringComparison.OrdinalIgnoreCase));
if (exclusion != null && (!exclusion.Crossgen2Only || runner.Index == CompilerIndex.CPAOT))
if (FrameworkExclusion.Exclude(simpleName, runner.Index, out string reason))
{
_frameworkExclusions[exclusion.SimpleName] = exclusion.Reason;
_frameworkExclusions[simpleName] = reason;
}
else
{
Expand All @@ -300,18 +272,21 @@ public bool CompileFramework()
foreach (string frameworkDll in ComputeManagedAssemblies.GetManagedAssembliesInFolder(_options.CoreRootDirectory.FullName))
{
string simpleName = Path.GetFileNameWithoutExtension(frameworkDll);
FrameworkExclusion exclusion = s_frameworkExclusions.FirstOrDefault(asm => asm.SimpleName.Equals(simpleName, StringComparison.OrdinalIgnoreCase));

ProcessInfo[] processes = new ProcessInfo[(int)CompilerIndex.Count];
compilationsPerRunner.Add(new KeyValuePair<string, ProcessInfo[]>(frameworkDll, processes));
foreach (CompilerRunner runner in frameworkRunners)
{
if (exclusion != null && (!exclusion.Crossgen2Only || runner.Index == CompilerIndex.CPAOT))
if (FrameworkExclusion.Exclude(simpleName, runner.Index, out string reason))
{
_frameworkExclusions[exclusion.SimpleName] = exclusion.Reason;
_frameworkExclusions[simpleName] = reason;
continue;
}
var compilationProcess = new ProcessInfo(new CompilationProcessConstructor(runner, _options.CoreRootDirectory.FullName, new string[] { frameworkDll }));
var compilationProcess = new ProcessInfo(
new CompilationProcessConstructor(
runner,
Path.Combine(runner.GetOutputPath(_options.CoreRootDirectory.FullName), Path.GetFileName(frameworkDll)),
new string[] { frameworkDll }));
compilationsToRun.Add(compilationProcess);
processes[(int)runner.Index] = compilationProcess;
}
Expand Down
58 changes: 57 additions & 1 deletion src/coreclr/src/tools/ReadyToRun.SuperIlc/CompilerRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;

Expand All @@ -19,6 +20,61 @@ public enum CompilerIndex
Count
}

public enum ExclusionType
{
Ignore,
DontCrossgen2,
}

public sealed class FrameworkExclusion
{
private static FrameworkExclusion[] s_list =
{
new FrameworkExclusion(ExclusionType.Ignore, "CommandLine", "Not a framework assembly"),
new FrameworkExclusion(ExclusionType.Ignore, "R2RDump", "Not a framework assembly"),
new FrameworkExclusion(ExclusionType.Ignore, "System.Runtime.WindowsRuntime", "WinRT is currently not supported"),
new FrameworkExclusion(ExclusionType.Ignore, "xunit.performance.api", "Not a framework assembly"),

// TODO (DavidWr): IBC-related failures
new FrameworkExclusion(ExclusionType.DontCrossgen2, "Microsoft.CodeAnalysis.CSharp", "Ibc TypeToken 6200019a has type token which resolves to a nil token"),
new FrameworkExclusion(ExclusionType.DontCrossgen2, "Microsoft.CodeAnalysis", "Ibc TypeToken 620001af unable to find external typedef"),
new FrameworkExclusion(ExclusionType.DontCrossgen2, "Microsoft.CodeAnalysis.VisualBasic", "Ibc TypeToken 620002ce unable to find external typedef"),
};

public readonly ExclusionType ExclusionType;
public readonly string SimpleName;
public readonly string Reason;

public FrameworkExclusion(ExclusionType exclusionType, string simpleName, string reason)
{
ExclusionType = exclusionType;
SimpleName = simpleName;
Reason = reason;
}

public static FrameworkExclusion Find(string simpleName)
{
return s_list.FirstOrDefault(fe => StringComparer.OrdinalIgnoreCase.Equals(simpleName, fe.SimpleName));
}

public static bool Exclude(string simpleName, CompilerIndex index, out string reason)
{
FrameworkExclusion exclusion = Find(simpleName);
if (exclusion != null &&
(exclusion.ExclusionType == ExclusionType.Ignore ||
exclusion.ExclusionType == ExclusionType.DontCrossgen2 && index == CompilerIndex.CPAOT))
{
reason = exclusion.Reason;
return true;
}
else
{
reason = null;
return false;
}
}
}

public abstract class CompilerRunner
{
/// <summary>
Expand Down Expand Up @@ -65,7 +121,7 @@ public virtual ProcessParameters CompilationProcess(string outputFileName, IEnum
}
else
{
processParameters.TimeoutMilliseconds = ProcessParameters.DefaultIlcTimeout;
processParameters.TimeoutMilliseconds = (_options.Composite ? ProcessParameters.DefaultIlcCompositeTimeout : ProcessParameters.DefaultIlcTimeout);
}
processParameters.LogPath = outputFileName + ".ilc.log";
processParameters.InputFileNames = inputAssemblyFileNames;
Expand Down
42 changes: 17 additions & 25 deletions src/coreclr/src/tools/ReadyToRun.SuperIlc/CpaotRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Runtime.InteropServices;

namespace ReadyToRun.SuperIlc
{

/// <summary>
/// Compiles assemblies using the Cross-Platform AOT compiler
/// </summary>
Expand All @@ -24,8 +23,6 @@ class CpaotRunner : CompilerRunner

private string Crossgen2Path => Path.Combine(_options.CoreRootDirectory.FullName, "crossgen2", "crossgen2.dll");

private List<string> _resolvedReferences;

public CpaotRunner(BuildOptions options, IEnumerable<string> referencePaths)
: base(options, referencePaths)
{
Expand Down Expand Up @@ -87,43 +84,38 @@ protected override IEnumerable<string> BuildCommandLineArguments(IEnumerable<str
yield return $"--parallelism={_options.Crossgen2Parallelism}";
}

char referenceOption = (_options.Composite ? 'u' : 'r');
HashSet<string> uniqueFolders = new HashSet<string>();
HashSet<string> uniqueFolders = new HashSet<string>(
RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
? StringComparer.OrdinalIgnoreCase
: StringComparer.Ordinal);
foreach (string assemblyFileName in assemblyFileNames)
{
uniqueFolders.Add(Path.GetDirectoryName(assemblyFileName));
}
foreach (string folder in uniqueFolders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(folder))
{
yield return $"-{referenceOption}:{reference}";
}
}

if (_resolvedReferences == null)
{
_resolvedReferences = ResolveReferences();
}
uniqueFolders.UnionWith(_referenceFolders);

foreach (string asmRef in _resolvedReferences)
foreach (string reference in ResolveReferences(uniqueFolders))
{
yield return asmRef;
yield return reference;
}
}

private List<string> ResolveReferences()
private IEnumerable<string> ResolveReferences(IEnumerable<string> folders)
{
char referenceOption = (_options.Composite ? 'u' : 'r');
List<string> references = new List<string>();
foreach (var referenceFolder in _referenceFolders)

foreach (string referenceFolder in folders)
{
foreach (var reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
foreach (string reference in ComputeManagedAssemblies.GetManagedAssembliesInFolder(referenceFolder))
{
references.Add($"-{referenceOption}:{reference}");
string simpleName = Path.GetFileNameWithoutExtension(reference);
if (!FrameworkExclusion.Exclude(simpleName, Index, out string reason))
{
yield return $"-{referenceOption}:{reference}";
}
}
}
return references;
}
}
}
5 changes: 5 additions & 0 deletions src/coreclr/src/tools/ReadyToRun.SuperIlc/ProcessRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,11 @@ public class ProcessParameters
/// </summary>
public const int DefaultIlcTimeout = 10 * 60 * 1000;

/// <summary>
/// Increase compilation timeout for composite builds.
/// </summary>
public const int DefaultIlcCompositeTimeout = 30 * 60 * 1000;

/// <summary>
/// Test execution timeout.
/// </summary>
Expand Down

0 comments on commit 17b28ba

Please sign in to comment.