Create test wrappers in run phase, not build phase aka make Crossgen2…

… runs green (dotnet#33898)

We need to compile the test wrappers in the test run phase as the
build phase is potentially common to multiple test runs and doesn't
generally know about execution details like the use of Crossgen
/ Crossgen2. Because of this Simon's change to baseline
Crossgen2-specific issues doesn't work.

Thanks

Tomas

eng/pipelines/coreclr/templates/build-test-job.yml

@@ -109,7 +109,7 @@ jobs:
 
 
     # Build managed test components
-    - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg)
       displayName: Build managed test components
 
 

eng/pipelines/coreclr/templates/run-test-job.yml

@@ -92,6 +92,55 @@ jobs:
         - name: LogNamePrefix
           value: TestRunLogs_R2R_CG2
 
+    # Set job timeouts
+    #
+    # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
+    # In case xUnit test wrappers get refactored this number should also be adjusted.
+    #
+    # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting
+    # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the
+    # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing.
+    #
+    # Please note that for Crossgen / Crossgen2 R2R runs, the "test running time" also includes the
+    # time needed to compile the test into native code with the Crossgen compiler.
+
+    - name: timeoutPerTestInMinutes
+      value: 10
+    - name: timeoutPerTestCollectionInMinutes
+      value: 30
+    - ${{ if eq(parameters.corefxTests, true) }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 10
+    - ${{ if in(parameters.testGroup, 'outerloop') }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 120
+    - ${{ if eq(parameters.crossgen2, true) }}:
+        - name: timeoutPerTestCollectionInMinutes
+          value: 90
+        - name: timeoutPerTestInMinutes
+          value: 30
+    - ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 360
+      # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times.
+      - name: timeoutPerTestInMinutes
+        value: 240
+    - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 120
+      - name: timeoutPerTestInMinutes
+        value: 30
+    - ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 240
+      - name: timeoutPerTestInMinutes
+        value: 60
+    - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}:
+      - name: timeoutPerTestCollectionInMinutes
+        value: 300
+      - name: timeoutPerTestInMinutes
+        value: 90
+
     - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
       - group: DotNet-HelixApi-Access
 
@@ -178,6 +227,11 @@ jobs:
         displayName: Generate test host
 
 
+    # Generate test wrappers
+    - script: $(coreClrRepoRootDir)build-test$(scriptExt) buildtestwrappersonly $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
+      displayName: Generate test wrappers
+
+
     # Crossgen framework assemblies prior to triggering readyToRun execution runs.
     - ${{ if eq(parameters.readyToRun, true) }}:
       - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg)
@@ -218,36 +272,8 @@ jobs:
 
         publishTestResults: true
 
-        # Set job timeouts
-        #
-        # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete.
-        # In case xUnit test wrappers get refactored this number should also be adjusted.
-        #
-        # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting
-        # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the
-        # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing.
-
-        ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
-          timeoutPerTestCollectionInMinutes: 120
-        ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}:
-          timeoutPerTestCollectionInMinutes: 30
-          timeoutPerTestInMinutes: 10
-        ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental') }}:
-          timeoutPerTestCollectionInMinutes: 120
-          timeoutPerTestInMinutes: 10
-        ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}:
-          timeoutPerTestCollectionInMinutes: 360
-          # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times.
-          timeoutPerTestInMinutes: 240
-        ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}:
-          timeoutPerTestCollectionInMinutes: 120
-          timeoutPerTestInMinutes: 30
-        ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}:
-          timeoutPerTestCollectionInMinutes: 240
-          timeoutPerTestInMinutes: 60
-        ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}:
-          timeoutPerTestCollectionInMinutes: 300
-          timeoutPerTestInMinutes: 90
+        timeoutPerTestInMinutes: $(timeoutPerTestInMinutes)
+        timeoutPerTestCollectionInMinutes: $(timeoutPerTestCollectionInMinutes)
 
         runCrossGen: ${{ and(eq(parameters.readyToRun, true), ne(parameters.crossgen2, true)) }}
         runCrossGen2: ${{ and(eq(parameters.readyToRun, true), eq(parameters.crossgen2, true)) }}

src/coreclr/build-test.cmd

@@ -52,6 +52,8 @@ set __CommonMSBuildArgs=
 
 set __SkipRestorePackages=
 set __SkipManaged=
+set __SkipTestWrappers=
+set __BuildTestWrappersOnly=
 set __SkipNative=
 set __RuntimeId=
 set __TargetsWindows=1
@@ -95,7 +97,9 @@ if /i "%1" == "ci"                    (set __ArcadeScriptArgs="-ci"&set __ErrMsg
 
 if /i "%1" == "skipmanaged"           (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "skipnative"            (set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "skiptestwrappers"      (set __SkipTestWrappers=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "buildtesthostonly"     (set __SkipNative=1&set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
+if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "buildagainstpackages"  (echo error: Remove /BuildAgainstPackages switch&&exit /b1)
 if /i "%1" == "skiprestorepackages"   (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
 if /i "%1" == "crossgen"              (set __DoCrossgen=1&set __TestBuildMode=crossgen&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop)
@@ -480,8 +484,12 @@ REM === Create test wrappers.
 REM ===
 REM =========================================================================================
 
+if defined __BuildTestWrappersOnly goto BuildTestWrappers
+
 if defined __SkipManaged goto SkipBuildingWrappers
+if defined __SkipTestWrappers goto SkipBuildingWrappers
 
+:BuildTestWrappers
 echo %__MsgPrefix%Creating test wrappers
 
 set __BuildLogRootName=Tests_XunitWrapper
@@ -514,6 +522,7 @@ REM ===
 REM =========================================================================================
 
 if defined __SkipCrossgenFramework goto SkipCrossgen
+if defined __BuildTestWrappersOnly goto SkipCrossgen
 
 set __CrossgenArg = ""
 if defined __DoCrossgen (

src/coreclr/build-test.sh

@@ -487,14 +487,18 @@ build_MSBuild_projects()
 }
 
 usage_list=("-buildtestwrappersonly: only build the test wrappers.")
+usage_list+=("-skiptestwrappers: Don't generate test wrappers.")
 usage_list+=("-copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests.")
 usage_list+=("-crossgen: Precompiles the framework managed assemblies in coreroot.")
+usage_list+=("-crossgen2: Precompiles the framework managed assemblies in coreroot using the Crossgen2 compiler.")
+usage_list+=("-generatetesthostonly: only generate the test host.")
 usage_list+=("-generatelayoutonly: only pull down dependencies and build coreroot.")
 usage_list+=("-priority1: include priority=1 tests in the build.")
 usage_list+=("-rebuild: if tests have already been built - rebuild them.")
 usage_list+=("-runtests: run tests after building them.")
-usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout.")
 usage_list+=("-skiprestorepackages: skip package restore.")
+usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout.")
+usage_list+=("-excludemonofailures: Mark the build as running on Mono runtime so that mono-specific issues are honored.")
 
 # Obtain the location of the bash script to figure out where the root of the repo is.
 __ProjectRoot="$(cd "$(dirname "$0")"; pwd -P)"
@@ -506,6 +510,10 @@ handle_arguments_local() {
             __BuildTestWrappersOnly=1
             ;;
 
+        skiptestwrappers|-skiptestwrappers)
+            __BuildTestWrappers=0
+            ;;
+
         copynativeonly|-copynativeonly)
             __SkipNative=1
             __SkipManaged=1

src/coreclr/tests/issues.targets

@@ -966,6 +966,9 @@
 
     <!-- Crossgen2 specific -->
     <ItemGroup Condition="'$(XunitTestBinBase)' != '' and '$(TestBuildMode)' == 'crossgen2' and '$(RuntimeFlavor)' == 'coreclr'">
+        <ExcludeList Include="$(XunitTestBinBase)/baseservices/threading/paramthreadstart/ThreadStartBool_1/*">
+            <Issue>https://github.com/dotnet/runtime/issues/33887</Issue>
+        </ExcludeList>
         <ExcludeList Include="$(XunitTestBinBase)/JIT/Intrinsics/TypeIntrinsics_r/*">
             <Issue>https://github.com/dotnet/runtime/issues/32725</Issue>
         </ExcludeList>

src/coreclr/tests/src/CLRTest.CrossGen.targets

@@ -71,7 +71,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then
           mkdir IL
           cp $(MSBuildProjectName).dll IL/$(MSBuildProjectName).dll
           mv $(MSBuildProjectName).dll $(MSBuildProjectName).org
-          __Command=$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/crossgen2/crossgen2.dll" -r:$CORE_ROOT/System.*.dll -r:$CORE_ROOT/Microsoft.*.dll -r:$CORE_ROOT/mscorlib.dll -r:$PWD/*.dll --targetarch=x64 -O --inputbubble $ExtraCrossGen2Args -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org
+          __Command=$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/crossgen2/crossgen2.dll" -r:$CORE_ROOT/System.*.dll -r:$CORE_ROOT/Microsoft.*.dll -r:$CORE_ROOT/mscorlib.dll -r:$CORE_ROOT/netstandard.dll -r:$PWD/*.dll --targetarch=x64 -O --inputbubble $ExtraCrossGen2Args -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org
           echo $__Command
           $__Command
           __cg2ExitCode=$?
@@ -131,7 +131,7 @@ if defined RunCrossGen2 (
             mkdir IL
             copy $(MSBuildProjectName).dll IL\$(MSBuildProjectName).dll
             ren $(MSBuildProjectName).dll $(MSBuildProjectName).org
-            set __Command=!_DebuggerFullPath! "!CORE_ROOT!\CoreRun.exe" "!CORE_ROOT!\crossgen2\crossgen2.dll" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble %21ExtraCrossGen2Args%21 -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:%25cd%25\*.dll
+            set __Command=!_DebuggerFullPath! "!CORE_ROOT!\CoreRun.exe" "!CORE_ROOT!\crossgen2\crossgen2.dll" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble %21ExtraCrossGen2Args%21 -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\netstandard.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:%25cd%25\*.dll
             echo "!__Command!"
             call !__Command!
             set CrossGen2Status=!ERRORLEVEL!