Skip to content

Commit

Permalink
Enable CoreFX test filtering for CoreCLR PRs (dotnet/coreclr#24259)
Browse files Browse the repository at this point in the history
This change enables CoreFX tests for CoreCLR. 


Commit migrated from dotnet/coreclr@b8f9262
  • Loading branch information
davmason authored May 3, 2019
1 parent 7ec8ff3 commit d52c238
Show file tree
Hide file tree
Showing 6 changed files with 636 additions and 51 deletions.
116 changes: 116 additions & 0 deletions eng/helixcorefxtests.proj
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
<Project InitialTargets="BuildHelixWorkItems" Sdk="Microsoft.DotNet.Helix.Sdk">
<Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />

<PropertyGroup>
<!-- Set the TargetFramework just to make the SDK happy -->
<TargetFramework>netstandard2.0</TargetFramework>
</PropertyGroup>

<PropertyGroup>
<Creator>$(_Creator)</Creator>
<HelixAccessToken>$(_HelixAccessToken)</HelixAccessToken>
<HelixBuild>$(_HelixBuild)</HelixBuild>
<HelixSource>$(_HelixSource)</HelixSource>
<HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
<HelixType>$(_HelixType)</HelixType>
<HelixArchitecture>$(__BuildArch)</HelixArchitecture>
<HelixConfiguration>$(__BuildType)</HelixConfiguration>

<!--
TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
-->
<ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory)..\</ProjectDir>
<RootBinDir Condition="'$(__RootBinDir)'==''">$(ProjectDir)bin\</RootBinDir>
<TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(RootBinDir)tests\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
<TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>

<TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
<TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>
<TestArchiveRuntimeFile>$(TestArchiveRuntimeRoot)testhost-runtime.zip</TestArchiveRuntimeFile>
<HelixCorrelationPayload></HelixCorrelationPayload>

<EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
<EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>
<EnableXUnitReporter>true</EnableXUnitReporter>
<FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
<FailOnWorkItemFailure>true</FailOnWorkItemFailure>

<TimeoutInSeconds Condition="'$(TimeoutInSeconds)' == ''">600</TimeoutInSeconds>
<CommandTimeoutSpan>$([System.TimeSpan]::FromSeconds($(TimeoutInSeconds)))</CommandTimeoutSpan>
<MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
<WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>
</PropertyGroup>

<PropertyGroup>
<!--
For windows we need to use call, since the command is going to be called from a bat script created by Helix
and we exit /b at the end of RunTests.cmd, Helix runs some other commands after ours within the bat script,
if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
The arguments passed in to the run script in order are the runtime directory, the dotnet root directory (for
helix submissions same as the runtime directory) and the global tools directory.
-->
<HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% -d %HELIX_CORRELATION_PAYLOAD% -g %HELIX_CORRELATION_PAYLOAD%\tools --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
<HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD -d $HELIX_CORRELATION_PAYLOAD -g $HELIX_CORRELATION_PAYLOAD/tools --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
</PropertyGroup>

<ItemGroup>
<TestArchiveRuntimeInputs Include="$(TestHostRootPath)**/*" />
</ItemGroup>

<Target Name="CopyRSPFile">
<Copy
SourceFiles="$(ProjectDir)\tests\CoreFX\CoreFX.issues.rsp"
DestinationFolder="$(TestHostRootPath)" />
</Target>

<Target Name="CompressRuntimeDirectory"
Inputs="@(TestArchiveRuntimeInputs)"
Outputs="$(TestArchiveRuntimeFile)"
DependsOnTargets="CopyRSPFile" >

<MakeDir Directories="$(TestArchiveRuntimeRoot)" />

<ZipDirectory
SourceDirectory="$(TestHostRootPath)"
DestinationFile="$(TestArchiveRuntimeFile)"
Overwrite="true" />

<Message Importance="High" Text="Zipped correlation payload into $(TestArchiveRuntimeFile)" />
</Target>

<PropertyGroup>
<TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
</PropertyGroup>

<Target Name="GetTestAssetManifest"
Returns="@(TestAssetBlobInfos)">

<PropertyGroup>
<_TargetGroup>netcoreapp</_TargetGroup>
<_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath>
</PropertyGroup>

<ParseBuildManifest AssetManifestPath="$(_AssetManifestPath)">
<Output TaskParameter="BlobInfos" ItemName="TestAssetBlobInfos" />
</ParseBuildManifest>
</Target>

<Target Name="BuildHelixWorkItems"
Inputs="$(TestArchiveRuntimeFile);@(TestAssetBlobInfos)"
Outputs="@(HelixCorrelationPayload);@(HelixWorkItem)"
DependsOnTargets="CompressRuntimeDirectory;GetTestAssetManifest">

<ItemGroup>
<HelixCorrelationPayload
Include="$(TestArchiveRuntimeFile)" />

<HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')">
<PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri>
<Command>$(HelixCommand)</Command>
<Timeout>$(CommandTimeoutSpan)</Timeout>
</HelixWorkItem>
</ItemGroup>

<Message Importance="High" Text="@(HelixWorkItem -> '%(PayloadUri)')" />
</Target>
</Project>
14 changes: 14 additions & 0 deletions eng/pipelines/coreclr/azure-pipelines.yml
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,18 @@ jobs:
jobParameters:
readyToRun: true
testGroup: innerloop
displayNameArgs: R2R
# The CoreFX runs against CoreCLR
- template: eng/platform-matrix.yml
parameters:
jobTemplate: test-job.yml
buildConfig: release
platforms:
- Windows_NT_x64
jobParameters:
corefxTests: true
testGroup: innerloop
displayNameArgs: CoreFX

# CI
- ${{ if and(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) }}:
Expand Down Expand Up @@ -263,6 +275,7 @@ jobs:
jobParameters:
readyToRun: true
testGroup: outerloop
displayNameArgs: R2R

#
# Release test builds
Expand Down Expand Up @@ -295,6 +308,7 @@ jobs:
jobParameters:
testGroup: outerloop
readyToRun: true
displayNameArgs: R2R


# Publish build information to Build Assets Registry
Expand Down
25 changes: 23 additions & 2 deletions eng/send-to-helix-step.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,17 @@ parameters:
timeoutPerTestCollectionInMinutes: ''
timeoutPerTestInMinutes: ''
runCrossGen: ''
helixProjectArguments: ''

steps:
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- powershell: eng\common\msbuild.ps1 -ci tests\helixpublishwitharcade.proj /maxcpucount
# TODO: Remove and consolidate this when we move to arcade via init-tools.cmd.
- powershell: eng\common\build.ps1 /p:DotNetPublishToBlobFeed=true -ci -restore -projects $(Build.SourcesDirectory)\eng\empty.proj
displayName: Restore blob feed tasks
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}

- powershell: eng\common\msbuild.ps1 -ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:$(Build.SourcesDirectory)/bin/Logs/SendToHelix.binlog
displayName: ${{ parameters.displayName }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
Expand All @@ -39,9 +46,21 @@ steps:
_TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
${{ if eq(parameters.publishTestResults, 'true') }}:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
# TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed
NUGET_PACKAGES: $(Build.SourcesDirectory)\.packages

- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- script: ./eng/common/msbuild.sh --ci tests/helixpublishwitharcade.proj /maxcpucount
# TODO: Remove and consolidate this when we move to arcade via init-tools.sh.
- script: ./eng/common/build.sh /p:DotNetPublishToBlobFeed=true --ci --restore --projects $(Build.SourcesDirectory)/eng/empty.proj
displayName: Restore blob feed tasks
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
${{ if eq(parameters.osGroup, 'FreeBSD') }}:
env:
# Arcade uses this SDK instead of trying to restore one.
DotNetCoreSdkDir: /usr/local/dotnet

- script: ./eng/common/msbuild.sh --ci ${{ parameters.helixProjectArguments }} /maxcpucount /bl:$(Build.SourcesDirectory)/bin/Logs/SendToHelix.binlog
displayName: ${{ parameters.displayName }}
${{ if ne(parameters.condition, '') }}:
condition: ${{ parameters.condition }}
Expand All @@ -62,3 +81,5 @@ steps:
_TimeoutPerTestInMinutes: ${{ parameters.timeoutPerTestInMinutes }}
${{ if eq(parameters.publishTestResults, 'true') }}:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
# TODO: remove NUGET_PACKAGES once https://github.com/dotnet/arcade/issues/1578 is fixed
NUGET_PACKAGES: $(Build.SourcesDirectory)/.packages
45 changes: 29 additions & 16 deletions eng/test-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ parameters:
readyToRun: false
helixQueues: ''
crossrootfsDir: ''
# If true, run the corefx tests instead of the coreclr ones
corefxTests: false
displayNameArgs: ''

### Test job

Expand All @@ -23,19 +26,13 @@ jobs:
helixType: 'build/tests/'

# Compute job name from template parameters
${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}:
name: ${{ format('testbuild_pri0_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri0 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
${{ if and(eq(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}:
name: ${{ format('testbuild_pri0_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri0 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, false)) }}:
name: ${{ format('testbuild_pri1_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri1 {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
${{ if and(ne(parameters.testGroup, 'innerloop'), eq(parameters.readyToRun, true)) }}:
name: ${{ format('testbuild_pri1_r2r_{0}_{1}_{2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri1 R2R {0} {1} {2}', parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
${{ if eq(parameters.testGroup, 'innerloop') }}:
name: ${{ format('testbuild_pri0_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri0 {0} {1} {2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

${{ if ne(parameters.testGroup, 'innerloop') }}:
name: ${{ format('testbuild_pri1_{0}_{1}_{2}_{3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}
displayName: ${{ format('Test Pri1 {0} {1} {2} {3}', parameters.displayNameArgs, parameters.osIdentifier, parameters.archType, parameters.buildConfig) }}

crossrootfsDir: ${{ parameters.crossrootfsDir }}

Expand Down Expand Up @@ -69,6 +66,16 @@ jobs:
- name: clangArg
value: '-clang5.0'

- name: testhostArg
value: ''
- ${{ if eq(parameters.corefxTests, true) }}:
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- name: testhostArg
value: 'generatetesthostonly'
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- name: testhostArg
value: 'buildtesthostonly'

# FreeBSD test jobs are disabled since we don't have any FreeBSD helix queues.
${{ if eq(parameters.osGroup, 'FreeBSD') }}:
condition: false
Expand Down Expand Up @@ -115,18 +122,18 @@ jobs:

# Build tests
- ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) $(clangArg)
- script: ./build-test.sh $(buildConfig) $(archType) $(crossArg) $(priorityArg) $(crossgenArg) $(clangArg) $(testhostArg)
displayName: Build tests
- ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
- script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg)
- script: build-test.cmd $(buildConfig) $(archType) $(priorityArg) $(crossgenArg) $(testhostArg)
displayName: Build tests


# Send tests to Helix
- template: /eng/send-to-helix-step.yml
parameters:
displayName: Send tests to Helix
buildConfig: ${{ parameters.buildConfig }}
buildConfig: $(buildConfigUpper)
archType: ${{ parameters.archType }}
osGroup: ${{ parameters.osGroup }}

Expand Down Expand Up @@ -172,6 +179,12 @@ jobs:
# DotNet-HelixApi-Access variable group
helixAccessToken: $(HelixApiAccessToken)

# Sets up the template to run the corefx tests
${{ if eq(parameters.corefxTests, true) }}:
helixProjectArguments: 'eng/helixcorefxtests.proj /t:Test /p:ArcadeBuild=True'
${{ if eq(parameters.corefxTests, false) }}:
helixProjectArguments: 'tests/helixpublishwitharcade.proj'

${{ if in(parameters.testGroup, 'innerloop', 'outerloop') }}:
scenarios:
- normal
Expand Down
Loading

0 comments on commit d52c238

Please sign in to comment.