From 58f931b1c5103c95c8d5f97213d95f265c83d6ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Rylek?= <trylek@microsoft.com>
Date: Mon, 27 Apr 2020 00:07:47 +0200
Subject: [PATCH] Optimize CoreCLR Helix runs by pruning the CORE_ROOT folder
 content (#35243)

This change limits the content of CORE_ROOT by removing some of the
unused artifacts, most notably the sharedFramework subfolder that
is not used by CoreCLR tests at all. The aim of this change is
to optimize the execution of CoreCLR tests in Helix where according
to Jarret's findings a large amount of time gets spent just
downloading and unpacking the test payloads.

Thanks

Tomas
---
 src/coreclr/build-test.cmd                    | 14 +++-----------
 src/coreclr/build-test.sh                     |  5 +----
 .../tests/src/Common/Directory.Build.targets  | 19 ++++++++++++++++++-
 3 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/src/coreclr/build-test.cmd b/src/coreclr/build-test.cmd
index ba8ddc2d6c5149..5f196486dbe201 100644
--- a/src/coreclr/build-test.cmd
+++ b/src/coreclr/build-test.cmd
@@ -157,6 +157,9 @@ set "__BinDir=%__RootBinDir%\bin\coreclr\%__TargetOS%.%__BuildArch%.%__BuildType
 set "__TestRootDir=%__RootBinDir%\tests\coreclr"
 set "__TestBinDir=%__TestRootDir%\%__TargetOS%.%__BuildArch%.%__BuildType%"
 
+if not defined XunitTestBinBase set XunitTestBinBase=%__TestBinDir%\
+set "CORE_ROOT=%XunitTestBinBase%\Tests\Core_Root"
+
 REM We have different managed and native intermediate dirs because the managed bits will include
 REM the configuration information deeper in the intermediates path.
 REM These variables are used by the msbuild project files.
@@ -290,9 +293,6 @@ if "%__SkipRestorePackages%" == "1" goto SkipRestoreProduct
 
 echo %__MsgPrefix%Restoring CoreCLR product from packages
 
-if not defined XunitTestBinBase set XunitTestBinBase=%__TestBinDir%\
-set "CORE_ROOT=%XunitTestBinBase%\Tests\Core_Root"
-
 set __BuildLogRootName=Restore_Product
 set __BuildLog=%__LogsDir%\%__BuildLogRootName%_%__TargetOS%__%__BuildArch%__%__BuildType%.log
 set __BuildWrn=%__LogsDir%\%__BuildLogRootName%_%__TargetOS%__%__BuildArch%__%__BuildType%.wrn
@@ -437,12 +437,8 @@ REM Remove any lock folder used for synchronization from previous runs.
 powershell -NoProfile "Get-ChildItem -path %__TestBinDir% -Include 'lock' -Recurse -Force |  where {$_.Attributes -eq 'Directory'}| Remove-Item -force -Recurse"
 
 set CORE_ROOT=%__TestBinDir%\Tests\Core_Root
-set CORE_ROOT_STAGE=%__TestBinDir%\Tests\Core_Root_Stage
 if exist "%CORE_ROOT%" rd /s /q "%CORE_ROOT%"
-if exist "%CORE_ROOT_STAGE%" rd /s /q "%CORE_ROOT_STAGE%"
 md "%CORE_ROOT%"
-md "%CORE_ROOT_STAGE%"
-xcopy /s "%__BinDir%" "%CORE_ROOT_STAGE%"
 
 REM =========================================================================================
 REM ===
@@ -480,8 +476,6 @@ if errorlevel 1 (
     exit /b 1
 )
 
-xcopy /s /y /i "%CORE_ROOT_STAGE%" "%CORE_ROOT%"
-
 REM =========================================================================================
 REM ===
 REM === Create test wrappers.
@@ -566,8 +560,6 @@ if defined __DoCrossgen2 (
 
 :SkipCrossgen
 
-rd /s /q "%CORE_ROOT_STAGE%"
-
 REM =========================================================================================
 REM ===
 REM === All builds complete!
diff --git a/src/coreclr/build-test.sh b/src/coreclr/build-test.sh
index c22c8ae949f790..5ab4a29e2fc5bc 100755
--- a/src/coreclr/build-test.sh
+++ b/src/coreclr/build-test.sh
@@ -119,13 +119,10 @@ generate_layout()
 
     mkdir -p "$CORE_ROOT"
 
-    build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/src/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
-
     chmod +x "$__BinDir"/corerun
     chmod +x "$__CrossgenExe"
 
-    # Make sure to copy over the pulled down packages
-    cp -r "$__BinDir"/* "$CORE_ROOT/" > /dev/null
+    build_MSBuild_projects "Tests_Overlay_Managed" "${__ProjectDir}/tests/src/runtest.proj" "Creating test overlay" "/t:CreateTestOverlay"
 
     if [[ "$__TargetOS" != "OSX" ]]; then
         nextCommand="\"$__TestDir/setup-stress-dependencies.sh\" --arch=$__BuildArch --outputDir=$CORE_ROOT"
diff --git a/src/coreclr/tests/src/Common/Directory.Build.targets b/src/coreclr/tests/src/Common/Directory.Build.targets
index 29fe3f80396494..dc5a0c59fc44e8 100644
--- a/src/coreclr/tests/src/Common/Directory.Build.targets
+++ b/src/coreclr/tests/src/Common/Directory.Build.targets
@@ -21,17 +21,34 @@
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName)%(Extension)')" />
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).ni%(Extension)')" />
       <RunTimeDependencyExcludeFiles Include="@(RunTimeDependencyExclude -> '%(FileName).pdb')" />
+
       <AllResolvedRuntimeDependencies Include="@(RuntimeCopyLocalItems -> '%(FileName)%(Extension)');@(NativeCopyLocalItems -> '%(FileName)%(Extension)')">
         <File>%(Identity)</File>
       </AllResolvedRuntimeDependencies>
+      
       <RunTimeDependencyCopyLocalFile Include="@(AllResolvedRuntimeDependencies)"  Exclude="@(RunTimeDependencyExcludeFiles)"/>
       <RunTimeDependencyCopyLocal Include="@(RunTimeDependencyCopyLocalFile -> '%(File)')"  />
       <RunTimeDependencyCopyLocal Include="$(TargetingPackPath)/*" />
+      
+      <RunTimeArtifactsExcludeFiles Include="protononjit.dll" />
+      <RunTimeArtifactsExcludeFiles Include="linuxonjit.dll" />
+      
+      <RunTimeArtifactsIncludeFolders Include="/" />
+      <RunTimeArtifactsIncludeFolders Include="IL/" />
+      <RunTimeArtifactsIncludeFolders Include="crossgen2/" />
+      <RunTimeArtifactsIncludeFolders Include="PDB/" />
+      <RunTimeArtifactsIncludeFolders Include="R2RTest/" />
+
+      <!-- Add binary dependencies to copy-local items -->
+      <RunTimeDependencyCopyLocal
+          Include="$(CoreCLRArtifactsPath)%(RunTimeArtifactsIncludeFolders.Identity)*"
+          Exclude="@(RunTimeArtifactsExcludeFiles -> '$(CoreCLRArtifactsPath)%(Identity)')"
+          TargetDir="%(RunTimeArtifactsIncludeFolders.Identity)" />
     </ItemGroup>
 
     <Copy
       SourceFiles="@(RunTimeDependencyCopyLocal)"
-      DestinationFolder="$(CORE_ROOT)"
+      DestinationFiles="@(RunTimeDependencyCopyLocal -> '$(CORE_ROOT)/%(TargetDir)%(Filename)%(Extension)')"
       SkipUnchangedFiles="$(SkipCopyUnchangedFiles)"
       OverwriteReadOnlyFiles="$(OverwriteReadOnlyFiles)"
       Retries="$(CopyRetryCount)"