diff --git a/eng/pipelines/coreclr/templates/build-test-job.yml b/eng/pipelines/coreclr/templates/build-test-job.yml index 59f06f78211a2..75650136520d5 100644 --- a/eng/pipelines/coreclr/templates/build-test-job.yml +++ b/eng/pipelines/coreclr/templates/build-test-job.yml @@ -109,7 +109,7 @@ jobs: # Build managed test components - - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg) + - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipnative skipgeneratelayout skiptestwrappers $(buildConfig) $(archType) $(crossArg) $(priorityArg) ci $(librariesOverrideArg) displayName: Build managed test components diff --git a/eng/pipelines/coreclr/templates/run-test-job.yml b/eng/pipelines/coreclr/templates/run-test-job.yml index 8be5c5eb49eab..4bdb1d38891bf 100644 --- a/eng/pipelines/coreclr/templates/run-test-job.yml +++ b/eng/pipelines/coreclr/templates/run-test-job.yml @@ -92,6 +92,55 @@ jobs: - name: LogNamePrefix value: TestRunLogs_R2R_CG2 + # Set job timeouts + # + # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. + # In case xUnit test wrappers get refactored this number should also be adjusted. + # + # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting + # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the + # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. + # + # Please note that for Crossgen / Crossgen2 R2R runs, the "test running time" also includes the + # time needed to compile the test into native code with the Crossgen compiler. + + - name: timeoutPerTestInMinutes + value: 10 + - name: timeoutPerTestCollectionInMinutes + value: 30 + - ${{ if eq(parameters.corefxTests, true) }}: + - name: timeoutPerTestCollectionInMinutes + value: 10 + - ${{ if in(parameters.testGroup, 'outerloop') }}: + - name: timeoutPerTestCollectionInMinutes + value: 120 + - ${{ if eq(parameters.crossgen2, true) }}: + - name: timeoutPerTestCollectionInMinutes + value: 90 + - name: timeoutPerTestInMinutes + value: 30 + - ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}: + - name: timeoutPerTestCollectionInMinutes + value: 360 + # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times. + - name: timeoutPerTestInMinutes + value: 240 + - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}: + - name: timeoutPerTestCollectionInMinutes + value: 120 + - name: timeoutPerTestInMinutes + value: 30 + - ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: + - name: timeoutPerTestCollectionInMinutes + value: 240 + - name: timeoutPerTestInMinutes + value: 60 + - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}: + - name: timeoutPerTestCollectionInMinutes + value: 300 + - name: timeoutPerTestInMinutes + value: 90 + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - group: DotNet-HelixApi-Access @@ -178,6 +227,11 @@ jobs: displayName: Generate test host + # Generate test wrappers + - script: $(coreClrRepoRootDir)build-test$(scriptExt) buildtestwrappersonly $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) + displayName: Generate test wrappers + + # Crossgen framework assemblies prior to triggering readyToRun execution runs. - ${{ if eq(parameters.readyToRun, true) }}: - script: $(coreClrRepoRootDir)build-test$(scriptExt) skipmanaged skipnative $(crossgenArg) $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(librariesOverrideArg) @@ -218,36 +272,8 @@ jobs: publishTestResults: true - # Set job timeouts - # - # "timeoutPerTestCollectionInMinutes" is the time needed for the "biggest" xUnit test collection to complete. - # In case xUnit test wrappers get refactored this number should also be adjusted. - # - # "timeoutPerTestInMinutes" corresponds to individual test running time. This is implemented by setting - # the __TestTimeout variable, which is later read by the coreclr xunit test wrapper code (the code in the - # xunit test dlls that invokes the actual tests). Note this doesn't apply to CoreFX testing. - - ${{ if and(eq(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: - timeoutPerTestCollectionInMinutes: 120 - ${{ if and(ne(parameters.corefxTests, true), eq(parameters.testGroup, 'innerloop')) }}: - timeoutPerTestCollectionInMinutes: 30 - timeoutPerTestInMinutes: 10 - ${{ if in(parameters.testGroup, 'outerloop', 'jit-experimental') }}: - timeoutPerTestCollectionInMinutes: 120 - timeoutPerTestInMinutes: 10 - ${{ if in(parameters.testGroup, 'gc-longrunning', 'gc-simulator') }}: - timeoutPerTestCollectionInMinutes: 360 - # gc reliability may take up to 2 hours to shutdown. Some scenarios have very long iteration times. - timeoutPerTestInMinutes: 240 - ${{ if in(parameters.testGroup, 'jitstress', 'jitstress-isas-arm', 'jitstress-isas-x86', 'jitstressregs-x86', 'jitstressregs', 'jitstress2-jitstressregs', 'jitelthookenabled' ) }}: - timeoutPerTestCollectionInMinutes: 120 - timeoutPerTestInMinutes: 30 - ${{ if in(parameters.testGroup, 'gcstress0x3-gcstress0xc') }}: - timeoutPerTestCollectionInMinutes: 240 - timeoutPerTestInMinutes: 60 - ${{ if in(parameters.testGroup, 'gcstress-extra', 'r2r-extra') }}: - timeoutPerTestCollectionInMinutes: 300 - timeoutPerTestInMinutes: 90 + timeoutPerTestInMinutes: $(timeoutPerTestInMinutes) + timeoutPerTestCollectionInMinutes: $(timeoutPerTestCollectionInMinutes) runCrossGen: ${{ and(eq(parameters.readyToRun, true), ne(parameters.crossgen2, true)) }} runCrossGen2: ${{ and(eq(parameters.readyToRun, true), eq(parameters.crossgen2, true)) }} diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd index e947e7a5038b0..b4cfa5161ea3b 100644 --- a/src/coreclr/build-test.cmd +++ b/src/coreclr/build-test.cmd @@ -52,6 +52,8 @@ set __CommonMSBuildArgs= set __SkipRestorePackages= set __SkipManaged= +set __SkipTestWrappers= +set __BuildTestWrappersOnly= set __SkipNative= set __RuntimeId= set __TargetsWindows=1 @@ -95,7 +97,9 @@ if /i "%1" == "ci" (set __ArcadeScriptArgs="-ci"&set __ErrMsg if /i "%1" == "skipmanaged" (set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "skipnative" (set __SkipNative=1&set __CopyNativeProjectsAfterCombinedTestBuild=false&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "skiptestwrappers" (set __SkipTestWrappers=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildtesthostonly" (set __SkipNative=1&set __SkipManaged=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) +if /i "%1" == "buildtestwrappersonly" (set __SkipNative=1&set __SkipManaged=1&set __BuildTestWrappersOnly=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "buildagainstpackages" (echo error: Remove /BuildAgainstPackages switch&&exit /b1) if /i "%1" == "skiprestorepackages" (set __SkipRestorePackages=1&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) if /i "%1" == "crossgen" (set __DoCrossgen=1&set __TestBuildMode=crossgen&set processedArgs=!processedArgs! %1&shift&goto Arg_Loop) @@ -480,8 +484,12 @@ REM === Create test wrappers. REM === REM ========================================================================================= +if defined __BuildTestWrappersOnly goto BuildTestWrappers + if defined __SkipManaged goto SkipBuildingWrappers +if defined __SkipTestWrappers goto SkipBuildingWrappers +:BuildTestWrappers echo %__MsgPrefix%Creating test wrappers set __BuildLogRootName=Tests_XunitWrapper @@ -514,6 +522,7 @@ REM === REM ========================================================================================= if defined __SkipCrossgenFramework goto SkipCrossgen +if defined __BuildTestWrappersOnly goto SkipCrossgen set __CrossgenArg = "" if defined __DoCrossgen ( diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh index 20755e3ded9d7..f4f2eecc631a0 100755 --- a/src/coreclr/build-test.sh +++ b/src/coreclr/build-test.sh @@ -487,14 +487,18 @@ build_MSBuild_projects() } usage_list=("-buildtestwrappersonly: only build the test wrappers.") +usage_list+=("-skiptestwrappers: Don't generate test wrappers.") usage_list+=("-copynativeonly: Only copy the native test binaries to the managed output. Do not build the native or managed tests.") usage_list+=("-crossgen: Precompiles the framework managed assemblies in coreroot.") +usage_list+=("-crossgen2: Precompiles the framework managed assemblies in coreroot using the Crossgen2 compiler.") +usage_list+=("-generatetesthostonly: only generate the test host.") usage_list+=("-generatelayoutonly: only pull down dependencies and build coreroot.") usage_list+=("-priority1: include priority=1 tests in the build.") usage_list+=("-rebuild: if tests have already been built - rebuild them.") usage_list+=("-runtests: run tests after building them.") -usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout.") usage_list+=("-skiprestorepackages: skip package restore.") +usage_list+=("-skipgeneratelayout: Do not generate the Core_Root layout.") +usage_list+=("-excludemonofailures: Mark the build as running on Mono runtime so that mono-specific issues are honored.") # Obtain the location of the bash script to figure out where the root of the repo is. __ProjectRoot="$(cd "$(dirname "$0")"; pwd -P)" @@ -506,6 +510,10 @@ handle_arguments_local() { __BuildTestWrappersOnly=1 ;; + skiptestwrappers|-skiptestwrappers) + __BuildTestWrappers=0 + ;; + copynativeonly|-copynativeonly) __SkipNative=1 __SkipManaged=1 diff --git a/src/coreclr/tests/issues.targets b/src/coreclr/tests/issues.targets index 59bee9cd51f97..27c9557369200 100644 --- a/src/coreclr/tests/issues.targets +++ b/src/coreclr/tests/issues.targets @@ -966,6 +966,9 @@ + + https://github.com/dotnet/runtime/issues/33887 + https://github.com/dotnet/runtime/issues/32725 diff --git a/src/coreclr/tests/src/CLRTest.CrossGen.targets b/src/coreclr/tests/src/CLRTest.CrossGen.targets index 5c9ba05e7147f..258f3078b6f92 100644 --- a/src/coreclr/tests/src/CLRTest.CrossGen.targets +++ b/src/coreclr/tests/src/CLRTest.CrossGen.targets @@ -71,7 +71,7 @@ if [ ! -z ${RunCrossGen2+x} ]%3B then mkdir IL cp $(MSBuildProjectName).dll IL/$(MSBuildProjectName).dll mv $(MSBuildProjectName).dll $(MSBuildProjectName).org - __Command=$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/crossgen2/crossgen2.dll" -r:$CORE_ROOT/System.*.dll -r:$CORE_ROOT/Microsoft.*.dll -r:$CORE_ROOT/mscorlib.dll -r:$PWD/*.dll --targetarch=x64 -O --inputbubble $ExtraCrossGen2Args -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org + __Command=$_DebuggerFullPath "$CORE_ROOT/corerun" "$CORE_ROOT/crossgen2/crossgen2.dll" -r:$CORE_ROOT/System.*.dll -r:$CORE_ROOT/Microsoft.*.dll -r:$CORE_ROOT/mscorlib.dll -r:$CORE_ROOT/netstandard.dll -r:$PWD/*.dll --targetarch=x64 -O --inputbubble $ExtraCrossGen2Args -o:$(scriptPath)$(MSBuildProjectName).dll $(scriptPath)$(MSBuildProjectName).org echo $__Command $__Command __cg2ExitCode=$? @@ -131,7 +131,7 @@ if defined RunCrossGen2 ( mkdir IL copy $(MSBuildProjectName).dll IL\$(MSBuildProjectName).dll ren $(MSBuildProjectName).dll $(MSBuildProjectName).org - set __Command=!_DebuggerFullPath! "!CORE_ROOT!\CoreRun.exe" "!CORE_ROOT!\crossgen2\crossgen2.dll" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble %21ExtraCrossGen2Args%21 -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:%25cd%25\*.dll + set __Command=!_DebuggerFullPath! "!CORE_ROOT!\CoreRun.exe" "!CORE_ROOT!\crossgen2\crossgen2.dll" %21scriptPath%21$(MSBuildProjectName).org -o:%21scriptPath%21$(MSBuildProjectName).dll --targetarch:x64 -O --inputbubble %21ExtraCrossGen2Args%21 -r:"!CORE_ROOT!\System.*.dll" -r:"!CORE_ROOT!\Microsoft.*.dll" -r:"!CORE_ROOT!\netstandard.dll" -r:"!CORE_ROOT!\mscorlib.dll" -r:%25cd%25\*.dll echo "!__Command!" call !__Command! set CrossGen2Status=!ERRORLEVEL!