GC long running tests (dotnet/coreclr#26489)

* - add gc-longrunning pipeline to run GC long running tests. - make GC reliability framework one of GC long running tests. * Temporarily enable Win10.ARM64 helix queue - for this PR only * increase timeouts * fewer inner cycles per one iteration in slower tests. * Final touches - increase GC reliability Framework timeout to 60min. remove Windows.10.Arm64.Open from the platform matrix (was added temporarily just to test this PR) * Disable ARM for now. Commit migrated from dotnet/coreclr@6b3b77a
shangte · Sep 5, 2019 · cd4e4f4 · cd4e4f4
1 parent 8e7dcaf
commit cd4e4f4
Show file tree

Hide file tree

Showing 11 changed files with 306 additions and 52 deletions.
diff --git a/eng/pipelines/coreclr/gc-longrunning.yml b/eng/pipelines/coreclr/gc-longrunning.yml
@@ -0,0 +1,27 @@
+trigger: none
+
+pr: none
+
+jobs:
+- template: /eng/platform-matrix.yml
+  parameters:
+    jobTemplate: build-job.yml
+    buildConfig: release
+    platforms:
+    - Linux_x64
+# disable ARM for now    - Linux_arm64
+    - Windows_NT_x64
+# disable ARM for now    - Windows_NT_arm64
+
+- template: /eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: release
+    platforms:
+    - Linux_x64
+# disable ARM for now    - Linux_arm64
+    - Windows_NT_x64
+# disable ARM for now    - Windows_NT_arm64
+    helixQueueGroup: ci
+    jobParameters:
+      testGroup: gc-longrunning
diff --git a/eng/run-test-job.yml b/eng/run-test-job.yml
@@ -62,6 +62,8 @@ jobs:
       timeoutInMinutes: 150
     ${{ if in(parameters.testGroup, 'outerloop') }}:
       timeoutInMinutes: 270
+    ${{ if in(parameters.testGroup, 'gc-longrunning') }}:
+      timeoutInMinutes: 480
     ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'gcstress0x3-gcstress0xc') }}:
       timeoutInMinutes: 390
     ${{ if in(parameters.testGroup, 'jitstress-isas-x86', 'gcstress-extra', 'r2r-extra') }}:
@@ -127,7 +129,11 @@ jobs:
           timeoutPerTestInMinutes: 10
         ${{ if in(parameters.testGroup, 'outerloop') }}:
           timeoutPerTestCollectionInMinutes: 120
-          timeoutPerTestInMinutes: 10
+          timeoutPerTestInMinutes: 10          
+        ${{ if in(parameters.testGroup, 'gc-longrunning') }}:
+          timeoutPerTestCollectionInMinutes: 360
+          # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times.
+          timeoutPerTestInMinutes: 240
         ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs' ) }}:
           timeoutPerTestCollectionInMinutes: 120
           timeoutPerTestInMinutes: 30
@@ -254,3 +260,7 @@ jobs:
           - jitminopts
           - forcerelocs
           - gcstress15
+        ${{ if in(parameters.testGroup, 'gc-longrunning') }}:
+          longRunningGcTests: true
+          scenarios:
+          - normal
diff --git a/eng/send-to-helix-step.yml b/eng/send-to-helix-step.yml
@@ -17,6 +17,7 @@ parameters:
   runCrossGen: ''
   helixProjectArguments: ''
   runInUnloadableContext: ''
+  longRunningGcTests: ''
 
 steps:
 - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
@@ -43,6 +44,7 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
+      _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _Scenarios: ${{ join(',', parameters.scenarios) }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
@@ -79,6 +81,7 @@ steps:
       _HelixType: ${{ parameters.helixType }}
       _RunCrossGen: ${{ parameters.runCrossGen }}
       _RunInUnloadableContext: ${{ parameters.runInUnloadableContext }}
+      _LongRunningGcTests: ${{ parameters.longRunningGcTests }}
       _Scenarios: ${{ join(',', parameters.scenarios) }}
       _TimeoutPerTestCollectionInMinutes: ${{ parameters.timeoutPerTestCollectionInMinutes }}
       _TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}

diff --git a/src/coreclr/tests/helixpublishwitharcade.proj b/src/coreclr/tests/helixpublishwitharcade.proj
@@ -25,6 +25,7 @@
         HelixType=$(_HelixType);
         PublishTestResults=$(_PublishTestResults);
         RunCrossGen=$(_RunCrossGen);
+        LongRunningGCTests=$(_LongRunningGCTests);
         RunInUnloadableContext=$(_RunInUnloadableContext);
         TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes);
         TimeoutPerTestInMinutes=$(_TimeoutPerTestInMinutes)
@@ -159,6 +160,8 @@
     <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFiles" StopOnFirstFailure="true" />
   </Target>
 
+
+
   <PropertyGroup>
     <EnableAzurePipelinesReporter>$(PublishTestResults)</EnableAzurePipelinesReporter>
     <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
@@ -169,19 +172,23 @@
     <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
     <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>
     <RunCrossGen Condition=" '$(RunCrossGen)' != 'true' ">false</RunCrossGen>
+    <LongRunningGCTests Condition=" '$(LongRunningGCTests)' != 'true' ">false</LongRunningGCTests>
     <TestRunNamePrefix Condition=" '$(RunCrossGen)' == 'true' ">R2R </TestRunNamePrefix>
     <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
     <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>
     <TimeoutPerTestInMilliseconds Condition=" '$(TimeoutPerTestInMinutes)' != '' ">$([System.TimeSpan]::FromMinutes($(TimeoutPerTestInMinutes)).TotalMilliseconds)</TimeoutPerTestInMilliseconds>
     <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
-    <XUnitRunnerArgs>-parallel collections -nocolor -noshadow -xml testResults.xml</XUnitRunnerArgs>
+    <_XUnitParallelMode>collections</_XUnitParallelMode>
+    <_XUnitParallelMode Condition=" '$(LongRunningGCTests)' == 'true' ">none</_XUnitParallelMode>
+    <XUnitRunnerArgs>-parallel $(_XUnitParallelMode) -nocolor -noshadow -xml testResults.xml</XUnitRunnerArgs>
   </PropertyGroup>
 
   <!-- WARNING: HelixPreCommand ItemGroup is intentionally minimal and should be kept that way. -->
 
   <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
     <HelixPreCommand Include="set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD%" />
     <HelixPreCommand Include="set RunCrossGen=1" Condition=" '$(RunCrossGen)' == 'true' " />
+    <HelixPreCommand Include="set RunningLongGCTests=1" Condition=" '$(LongRunningGCTests)' == 'true' " />
     <HelixPreCommand Include="set RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="set CLRCustomTestLauncher=%HELIX_CORRELATION_PAYLOAD%\runincontext.cmd" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="set __TestEnv=%HELIX_WORKITEM_PAYLOAD%\$(TestEnvFileName)" />
@@ -192,6 +199,7 @@
   <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
     <HelixPreCommand Include="export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD" />
     <HelixPreCommand Include="export RunCrossGen=1" Condition=" '$(RunCrossGen)' == 'true' " />
+    <HelixPreCommand Include="export RunningLongGCTests=1" Condition=" '$(LongRunningGCTests)' == 'true' " />
     <HelixPreCommand Include="export RunInUnloadableContext=1" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="export CLRCustomTestLauncher=$HELIX_CORRELATION_PAYLOAD/runincontext.sh" Condition=" '$(RunInUnloadableContext)' == 'true' " />
     <HelixPreCommand Include="export __TestEnv=$HELIX_WORKITEM_PAYLOAD/$(TestEnvFileName)" />

diff --git a/src/coreclr/tests/src/GC/Stress/Framework/LoaderClass.cs b/src/coreclr/tests/src/GC/Stress/Framework/LoaderClass.cs
@@ -146,7 +146,7 @@ public void Load(string assemblyName, string[] paths
 #if !PROJECTK_BUILD
             assem = Assembly.Load(an);
 #else
-            LoadFrom(assemblyName + ".exe", paths);
+            LoadFrom(assemblyName + ".dll", paths);
 #endif
         }
         catch
@@ -200,7 +200,7 @@ public Object GetTest()
         }
         catch (System.Reflection.ReflectionTypeLoadException e)
         {
-            if (assembly.ToLower().IndexOf(".exe") != -1)
+            if (assembly.ToLower().IndexOf(".dll") != -1)
                 return (assembly);
             throw new Exception(String.Format("Couldn't GetTypes for {0} ({1})", assembly, e.Message));
         }

diff --git a/src/coreclr/tests/src/GC/Stress/Framework/ReliabilityConfiguration.cs b/src/coreclr/tests/src/GC/Stress/Framework/ReliabilityConfiguration.cs
@@ -187,7 +187,7 @@ private bool GetTrueFalseOptionValue(string value, string configSettingName)
         throw new Exception(String.Format("Unknown option value for {0}: {1}", configSettingName, value));
     }
 
-
+    // returns time in minutes
     public static int ConvertTimeValueToTestRunTime(string timeValue)
     {
         int returnValue;
@@ -381,7 +381,15 @@ private void GetTestsToRun(string testConfig)
 
                                 _curTestSet = new ReliabilityTestSet();
 
-                                while (currentXML.MoveToNextAttribute())
+                                // when running as an ordinary test, limit run time to 10 min.
+                                bool limitTime = ReliabilityFramework.IsRunningAsUnitTest && !ReliabilityFramework.IsRunningLongGCTests;
+
+                                if (limitTime)
+                                {
+                                    _curTestSet.MaximumTime = 10;
+                                }
+
+                                    while (currentXML.MoveToNextAttribute())
                                 {
                                     XmlDebugOut(" " + currentXML.Name + "=\"" + currentXML.Value + "\"");
                                     switch (currentXML.Name)
@@ -391,7 +399,12 @@ private void GetTestsToRun(string testConfig)
                                             break;
                                         case "maximumExecutionTime":
                                             string timeValue = currentXML.Value;
-                                            _curTestSet.MaximumTime = ConvertTimeValueToTestRunTime(timeValue);
+
+                                            if (!limitTime)
+                                            {
+                                                _curTestSet.MaximumTime = ConvertTimeValueToTestRunTime(timeValue);
+                                            }
+
                                             break;
                                         case "id":
                                             _curTestSet.FriendlyName = currentXML.Value;

diff --git a/src/coreclr/tests/src/GC/Stress/Framework/ReliabilityFramework.cs b/src/coreclr/tests/src/GC/Stress/Framework/ReliabilityFramework.cs
@@ -78,7 +78,7 @@ protected override Assembly Load(AssemblyName assemblyName)
         else
         {
             Console.WriteLine("CustomAssemblyLoader: this looks like a test");
-            strPath = Path.Combine(_testsPath, assemblyName.Name + ".exe");
+            strPath = Path.Combine(_testsPath, assemblyName.Name + ".dll");
         }
 
         Console.WriteLine("Incoming AssemblyName: {0}", assemblyName.ToString());
@@ -147,6 +147,10 @@ public class ReliabilityFramework
     // constants
     private const string waitingText = "Waiting for all tests to finish loading, Remaining Tests: ";
 
+    // support for running in automation
+    internal static bool IsRunningAsUnitTest = false;
+    internal static bool IsRunningLongGCTests = false;
+
     /// <summary>
     /// Our main execution routine for the reliability framework.  Here we create an instance of the framework & run the reliability tests
     /// in it.  All code in here will execute in our starting app domain.
@@ -173,6 +177,10 @@ public static int Main(string[] args)
                 {
                     doReplay = true;
                 }
+                else if (String.Compare(arg.Substring(1), "unittest", true) == 0)
+                {
+                    IsRunningAsUnitTest = true;
+                }
                 else if (String.Compare(arg.Substring(1, arg.IndexOf(':') - 1), sTests, true) == 0)
                 {
                     String testlist = arg.Substring(sTests.Length + 2);
@@ -188,6 +196,7 @@ public static int Main(string[] args)
                 {
                     timeValue = arg.Substring(exectime.Length + 2);
                 }
+
                 else
                 {
                     Console.WriteLine("Unknown option: {0}", arg);
@@ -199,12 +208,28 @@ public static int Main(string[] args)
                 configFile = arg;
             }
         }
+
+        IsRunningLongGCTests = System.Environment.GetEnvironmentVariable("RunningLongGCTests") == "1";
+
+        // if no config file specified, check for [something]_gc.config in the current folder.
+        if (configFile == null)
+        {
+            var config = IsRunningAsUnitTest ? 
+                "*_gc_ci.config" : 
+                "*_gc.config";
+
+            configFile = Directory.GetFiles(Environment.CurrentDirectory, config).SingleOrDefault();
+        }
+
         if (configFile == null)
         {
             okToContinue = false;
             Console.WriteLine("You must specify a config file!");
             rf._logger.WriteToInstrumentationLog(null, LoggingLevels.StartupShutdown, "No configuration file specified.");
         }
+
+        System.Console.WriteLine("Using config file: " + configFile);
+
         if (!okToContinue)
         {
             Console.WriteLine("\r\nHost Interface Reliability Harness\r\n");
@@ -215,6 +240,7 @@ public static int Main(string[] args)
             Console.WriteLine(" -replay     -   Replay from log file");
             Console.WriteLine(" -{0}:<tests>	-	Comma delimited list of tests to run (no spaces)", sTests);
             Console.WriteLine(" -{0}:<seed>	-	Random Number seed for replays", sSeed);
+            Console.WriteLine(" -unittest   -   Set when run via unit test harness");
             rf._logger.WriteToInstrumentationLog(null, LoggingLevels.StartupShutdown, "Not ok to continue.");
 
 #if PROJECTK_BUILD
@@ -250,11 +276,18 @@ public static int Main(string[] args)
 
                     eTemp = e.InnerException;
                 }
+
+                string err = String.Format("Exception while running tests: {0}", e);
+
                 if (eTemp == null)
                 {
-                    rf._logger.WriteToInstrumentationLog(null, LoggingLevels.Tests, String.Format("Exception while running tests: {0}", e));
+                    rf._logger.WriteToInstrumentationLog(null, LoggingLevels.Tests, err);
                     Console.WriteLine("There was an exception while attempting to run the tests: See Instrumentation Log for details. (Exception: {0})", e);
                 }
+
+                // crash on exceptions when running as a unit test.
+                if (IsRunningAsUnitTest)
+                    Environment.FailFast(err, e);
             }
         }
         finally
@@ -1182,6 +1215,10 @@ private void StartTestWorker(object test)
                         }
                         catch (Exception e)
                         {
+                            // crash on exceptions when running as a unit test.
+                            if (IsRunningAsUnitTest)
+                                Environment.FailFast("Test failed", e);
+
                             Console.WriteLine(e);
                         }
                         Interlocked.Increment(ref _testsRanCount);
@@ -1420,12 +1457,18 @@ private void StartTestWorker(object test)
                                     eTemp = eTemp.InnerException;
                                 }
 
+                                string err = String.Format("Error in executing test {0}: {1}", daTest.RefOrID, e);
+
                                 if (eTemp == null)
                                 {
-                                    _logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Error in executing test {0}: {1}", daTest.RefOrID, e));
+                                    _logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, err);
                                     AddFailure("Failed to ExecuteAssembly (" + e.ToString() + ")", daTest, -1);
                                 }
 
+                                // crash on exceptions when running as a unit test.
+                                if (IsRunningAsUnitTest)
+                                    Environment.FailFast(err, e);
+
 #if !PROJECTK_BUILD
                                 if ((Thread.CurrentThread.ThreadState & System.Threading.ThreadState.AbortRequested) != 0)
                                 {
@@ -1452,8 +1495,14 @@ private void StartTestWorker(object test)
                             }
                             catch (Exception e)
                             {
-                                Console.WriteLine("Error in executing ISingleReliabilityTest: {0}", e);
+                                string err = $"Error in executing ISingleReliabilityTest: {e}";
+
+                                Console.WriteLine(err);
                                 AddFailure("ISingleReliabilityTest threw exception!", daTest, -1);
+
+                                // crash on exceptions when running as a unit test.
+                                if (IsRunningAsUnitTest)
+                                    Environment.FailFast(err, e);
                             }
                         }
                         else if (daTest.TestObject is IMultipleReliabilityTest)
@@ -1471,8 +1520,14 @@ private void StartTestWorker(object test)
                             }
                             catch (Exception ex)
                             {
-                                Console.WriteLine("Error in executing IMultipleReliabilityTest: {0}", ex);
+                                string err = $"Error in executing IMultipleReliabilityTest: {ex}";
+
+                                Console.WriteLine(err);
                                 AddFailure("IMultipleReliabilityTest threw exception!", daTest, -1);
+
+                                // crash on exceptions when running as a unit test.
+                                if (IsRunningAsUnitTest)
+                                    Environment.FailFast(err, ex);
                             }
                         }
 
@@ -1569,7 +1624,12 @@ private void StartTestWorker(object test)
         }
         catch (Exception e)
         {
-            _logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Unexpected exception on StartTestWorker: {0}", e));
+            string err = String.Format("Unexpected exception on StartTestWorker: {0}", e);
+            _logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, err);
+
+            // crash on exceptions when running as a unit test.
+            if (IsRunningAsUnitTest)
+                Environment.FailFast(err, e);
         }
     }
 
@@ -1701,6 +1761,10 @@ private void TestPreLoader(ReliabilityTest test, string[] paths)
             {
                 BadTestDebugBreak(msg);
             }
+
+            // crash on exceptions when running as a unit test.
+            if (IsRunningAsUnitTest)
+                Environment.FailFast(msg, e);
         }
         Interlocked.Decrement(ref LoadingCount);
     }