Skip to content

Commit

Permalink
Optimize CoreCLR Helix runs by pruning the CORE_ROOT folder content (d…
Browse files Browse the repository at this point in the history
…otnet#35243)

This change limits the content of CORE_ROOT by removing some of the
unused artifacts, most notably the sharedFramework subfolder that
is not used by CoreCLR tests at all. The aim of this change is
to optimize the execution of CoreCLR tests in Helix where according
to Jarret's findings a large amount of time gets spent just
downloading and unpacking the test payloads.

Thanks

Tomas
trylek authored Apr 26, 2020
1 parent b93693e commit 58f931b
Showing 3 changed files with 22 additions and 16 deletions.
14 changes: 3 additions & 11 deletions src/coreclr/build-test.cmd
Original file line number Diff line number Diff line change
@@ -157,6 +157,9 @@ set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOS%.%__BuildArch%.%__BuildType
set "__TestRootDir=%__RootBinDir%\tests\coreclr"
set "__TestBinDir=%__TestRootDir%\%__TargetOS%.%__BuildArch%.%__BuildType%"

if not defined XunitTestBinBase set XunitTestBinBase=%__TestBinDir%\
set "CORE_ROOT=%XunitTestBinBase%\Tests\Core_Root"

REM We have different managed and native intermediate dirs because the managed bits will include
REM the configuration information deeper in the intermediates path.
REM These variables are used by the msbuild project files.
@@ -290,9 +293,6 @@ if "%__SkipRestorePackages%" == "1" goto SkipRestoreProduct

echo %__MsgPrefix%Restoring CoreCLR product from packages

if not defined XunitTestBinBase set XunitTestBinBase=%__TestBinDir%\
set "CORE_ROOT=%XunitTestBinBase%\Tests\Core_Root"

set __BuildLogRootName=Restore_Product
set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__TargetOS%__%__BuildArch%__%__BuildType%.log
set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__TargetOS%__%__BuildArch%__%__BuildType%.wrn
@@ -437,12 +437,8 @@ REM Remove any lock folder used for synchronization from previous runs.
powershell -NoProfile "Get-ChildItem -path %__TestBinDir% -Include 'lock' -Recurse -Force | where {$_.Attributes -eq 'Directory'}| Remove-Item -force -Recurse"

set CORE_ROOT=%__TestBinDir%\Tests\Core_Root
set CORE_ROOT_STAGE=%__TestBinDir%\Tests\Core_Root_Stage
if exist "%CORE_ROOT%" rd /s /q "%CORE_ROOT%"
if exist "%CORE_ROOT_STAGE%" rd /s /q "%CORE_ROOT_STAGE%"
md "%CORE_ROOT%"
md "%CORE_ROOT_STAGE%"
xcopy /s "%__BinDir%" "%CORE_ROOT_STAGE%"

REM =========================================================================================
REM ===
@@ -480,8 +476,6 @@ if errorlevel 1 (
exit /b 1
)

xcopy /s /y /i "%CORE_ROOT_STAGE%" "%CORE_ROOT%"

REM =========================================================================================
REM ===
REM === Create test wrappers.
@@ -566,8 +560,6 @@ if defined __DoCrossgen2 (

:SkipCrossgen

rd /s /q "%CORE_ROOT_STAGE%"

REM =========================================================================================
REM ===
REM === All builds complete!
5 changes: 1 addition & 4 deletions src/coreclr/build-test.sh
Original file line number Diff line number Diff line change
@@ -119,13 +119,10 @@ generate_layout()

mkdir -p "$CORE_ROOT"

build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/src/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"

chmod +x "$__BinDir"/corerun
chmod +x "$__CrossgenExe"

# Make sure to copy over the pulled down packages
cp -r "$__BinDir"/* "$CORE_ROOT/" > /dev/null
build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/src/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"

if [[ "$__TargetOS" != "OSX" ]]; then
nextCommand="\"$__TestDir/setup-stress-dependencies.sh\" --arch=$__BuildArch --outputDir=$CORE_ROOT"
19 changes: 18 additions & 1 deletion src/coreclr/tests/src/Common/Directory.Build.targets
Original file line number Diff line number Diff line change
@@ -21,17 +21,34 @@
<RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName)%(Extension)')" />
<RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).ni%(Extension)')" />
<RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).pdb')" />

<AllResolvedRuntimeDependencies Include="@(RuntimeCopyLocalItems -> '%(FileName)%(Extension)');@(NativeCopyLocalItems -> '%(FileName)%(Extension)')">
<File>%(Identity)</File>
</AllResolvedRuntimeDependencies>

<RunTimeDependencyCopyLocalFile Include="@(AllResolvedRuntimeDependencies)" Exclude="@(RunTimeDependencyExcludeFiles)"/>
<RunTimeDependencyCopyLocal Include="@(RunTimeDependencyCopyLocalFile -> '%(File)')" />
<RunTimeDependencyCopyLocal Include="$(TargetingPackPath)/*" />

<RunTimeArtifactsExcludeFiles Include="protononjit.dll" />
<RunTimeArtifactsExcludeFiles Include="linuxonjit.dll" />

<RunTimeArtifactsIncludeFolders Include="/" />
<RunTimeArtifactsIncludeFolders Include="IL/" />
<RunTimeArtifactsIncludeFolders Include="crossgen2/" />
<RunTimeArtifactsIncludeFolders Include="PDB/" />
<RunTimeArtifactsIncludeFolders Include="R2RTest/" />

<!-- Add binary dependencies to copy-local items -->
<RunTimeDependencyCopyLocal
Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)*"
Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
</ItemGroup>

<Copy
SourceFiles="@(RunTimeDependencyCopyLocal)"
DestinationFolder="$(CORE_ROOT)"
DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(Filename)%(Extension)')"
SkipUnchangedFiles="$(SkipCopyUnchangedFiles)"
OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
Retries="$(CopyRetryCount)"

0 comments on commit 58f931b

Please sign in to comment.