helixcorefxtests.proj

<Project InitialTargets="BuildHelixWorkItems" DefaultTargets="RunInParallelForEachScenario">

  <!-- This project uses the helix SDK, documented at
       https://github.com/dotnet/arcade/tree/master/src/Microsoft.DotNet.Helix/Sdk,
       to send test jobs to helix. -->

  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.props" Condition=" '$(UsesHelixSdk)' == 'true' " />
  <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.props))" Condition=" '$(UsesHelixSdk)' != 'true' "/>

  <!-- Import Microsoft.DotNet.Build.Tasks.Feed.targets for `ParseBuildManifest` -->
  <Import Project="$(NuGetPackageRoot)microsoft.dotnet.build.tasks.feed\$(MicrosoftDotNetBuildTasksFeedVersion)\build\Microsoft.DotNet.Build.Tasks.Feed.targets" />

  <!-- This project is copies much logic from helixpublishwitharcade.proj, used to send coreclr tests to Helix. -->

  <!--
    "RunInParallelForEachScenario" is the "root" target for this Project. It invokes other targets to set up the
    information needed to submit a job to Helix. It does so by "recursively" invoking this project file with
    different properties and targets to run, as well as invoking MSBuild on other project files. In particular,
    it first creates the "correlation payload directory", which is the directory of files used by all Helix
    submissions. Then, it recursively invokes this Project once per scenario (e.g., normal Pri-1 run, JitStress=2
    run, JitStressRegs=8 run, etc.), creating a new set of Helix work items for each scenario. For the CoreFX
    tests, we create one Helix work item for each CoreFX test assembly; there is currently no concept of
    grouping multiple assemblies into a single Helix work item to, for instance, consolidate multiple short-running
    assemblies together. (Note that this grouping is done for the CoreCLR tests in helixpublishwitharcade.proj.)
  -->
  <Target Name="RunInParallelForEachScenario">
    <PropertyGroup>
      <!-- This specifies what properties are needed to be passed down as global properties to a child project. -->

      <_PropertiesToPass>
        __BuildArch=$(__BuildArch);
        __BuildOS=$(__BuildOS);
        __BuildType=$(__BuildType);
        Creator=$(_Creator);
        HelixAccessToken=$(_HelixAccessToken);
        HelixBuild=$(_HelixBuild);
        HelixSource=$(_HelixSource);
        HelixTargetQueues=$(_HelixTargetQueues);
        HelixType=$(_HelixType);
        PublishTestResults=$(_PublishTestResults);
        RunCrossGen=$(_RunCrossGen);
        TimeoutPerTestCollectionInMinutes=$(_TimeoutPerTestCollectionInMinutes)
      </_PropertiesToPass>
    </PropertyGroup>

    <Error Condition="'$(_Scenarios)' == ''" Text="_Scenarios not set" />

    <MSBuild Projects="$(MSBuildProjectFile)" Targets="PrepareCorrelationPayloadDirectory" Properties="Scenarios=$(_Scenarios)" />

    <ItemGroup>
      <_Scenarios Include="$(_Scenarios.Split(','))" />

      <!-- MSBuild creates a new instance of the project for each %(_Scenarios.Identity) and can build them in parallel. -->
      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
        <AdditionalProperties>$(_PropertiesToPass);Scenario=%(_Scenarios.Identity)</AdditionalProperties>
      </_ProjectsToBuild>
    </ItemGroup>

    <PropertyGroup>
      <_BuildInParallel>false</_BuildInParallel>
      <_BuildInParallel Condition=" '@(_ProjectsToBuild->Count())' &gt; '1' ">true</_BuildInParallel>
    </PropertyGroup>

    <!-- Invoke MSBuild on this project file once for each Scenario (because of the "batching" defined in "_ProjectsToBuild").
         Set "UsesHelixSdk=true" to indicate we want to invoke Helix on this invocation: create the Helix work items and start
         the jobs. This is done by invoking the "Test" Helix target.
    -->
    <MSBuild Projects="@(_ProjectsToBuild)" Targets="Test" BuildInParallel="$(_BuildInParallel)" StopOnFirstFailure="false" Properties="UsesHelixSdk=true" />
  </Target>

  <!-- Define a set of properties that are input to the Project, and that must be passed down to child processes.
      (See "_PropertiesToPass" in target "RunInParallelForEachScenario".) If something can be computed, such as
      properties needed by Helix, they should probably be set in target "BuildHelixWorkItems" instead.
  -->

  <PropertyGroup>
    <Creator>$(_Creator)</Creator>
    <HelixAccessToken>$(_HelixAccessToken)</HelixAccessToken>
    <HelixBuild>$(_HelixBuild)</HelixBuild>
    <HelixSource>$(_HelixSource)</HelixSource>
    <HelixTargetQueues>$(_HelixTargetQueues)</HelixTargetQueues>
    <HelixType>$(_HelixType)</HelixType>

    <!--
      TODO: ProjectDir, RootBinDir, TestWorkingDir, and TargetsWindows are global properties set in dir.props, remove the property assignment here when we port to arcade.
     -->
    <ProjectDir Condition="'$(__ProjectDir)'==''">$(MSBuildThisFileDirectory)..\</ProjectDir>
    <RootBinDir Condition="'$(__RootBinDir)'==''">$(ProjectDir)artifacts\bin\coreclr</RootBinDir>
    <TestWorkingDir Condition="'$(__TestWorkingDir)'==''">$(ProjectDir)artifacts\tests\coreclr\$(__BuildOS).$(__BuildArch).$(__BuildType)\</TestWorkingDir>
    <TargetsWindows Condition="'$(__BuildOS)' == 'Windows_NT'">true</TargetsWindows>

    <!-- We have a single, universal exclusion file for all CoreFX test exclusions, for CoreFX tests run
         in the CoreCLR repo. Note in particular that we don't have a unique exclusion file for each test
         assembly, or for each processor architecture, or for each operating system, or for each stress
         scenario type. This means that we generally exclude more tests than is strictly necessary for
         any particular job. However, this mechanism is super simple; we don't need to manage fine-
         grained exclusions; and there really shouldn't be any (or many) exclusions anyway.
    -->
    <CoreFXTestExclusionFile>$(ProjectDir)src\coreclr\tests\CoreFX\CoreFX.issues.rsp</CoreFXTestExclusionFile>

    <TestHostRootPath>$(TestWorkingDir)testhost\</TestHostRootPath>
    <TestArchiveRuntimeRoot>$(TestWorkingDir)helix\</TestArchiveRuntimeRoot>

    <!-- The VM PDB symbols for Windows live in the same directory as the VM binaries. Set a relative
         path within TestHostRootPath (root of the correlation payload) that we can use to set the
         symbol path for the purpose of getting a good stack trace for debug build VM asserts.
    -->
    <TestHostPDBPath>shared\$(MicrosoftNETCoreAppPackage)\$(ProductVersion)</TestHostPDBPath>

    <TestAssetBlobFeedUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core</TestAssetBlobFeedUrl>
  </PropertyGroup>

  <PropertyGroup>
    <!--
      For Windows, we need to use "call", since the command is going to be called from a batch script created by Helix.
      We "exit /b" at the end of RunTests.cmd. Helix runs some other commands after ours within the batch script,
      so if we don't use call, then we cause the parent script to exit, and anything after will not be executed.
      The arguments passed in to the run script in are the runtime directory (-r) and the exclusion response file
      passed to xunit (rsp-file).
    -->
    <HelixCommand Condition="'$(TargetsWindows)' == 'true'">call RunTests.cmd -r %HELIX_CORRELATION_PAYLOAD% --rsp-file %HELIX_CORRELATION_PAYLOAD%\CoreFX.issues.rsp </HelixCommand>
    <HelixCommand Condition="'$(TargetsWindows)' != 'true'">./RunTests.sh -r $HELIX_CORRELATION_PAYLOAD --rsp-file $HELIX_CORRELATION_PAYLOAD/CoreFX.issues.rsp </HelixCommand>
  </PropertyGroup>

  <!--
    Copy the CoreFX test exclusion response file (which is passed to the xunit console runner) to someplace we can
    access it during the test run. We currently just copy it to the root of the generated .NET Core testhost
    directory, which we use as the root of the correlation directory.
  -->
  <Target Name="CopyRSPFile">
    <Copy
      SourceFiles="$(CoreFXTestExclusionFile)"
      DestinationFolder="$(TestHostRootPath)" />

    <Message Importance="High" Text="Copied $(CoreFXTestExclusionFile) into $(TestHostRootPath)" />
  </Target>

  <PropertyGroup>
    <!-- Set the name of the scenario file. Note that this is only used in invocations where $(Scenario) is set. -->
    <TestEnvFileName Condition=" '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
    <TestEnvFileName Condition=" '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
  </PropertyGroup>

  <Target Name="CreateTestEnvFile">
    <!-- This target creates one __TestEnv file for the single $(Scenario). -->

    <PropertyGroup>
      <TestEnvFilePath>$(TestHostRootPath)$(TestEnvFileName)</TestEnvFilePath>
    </PropertyGroup>

    <ItemGroup>
      <_ProjectsToBuild Include="$(ProjectDir)src\coreclr\tests\testenvironment.proj">
        <Properties>Scenario=$(Scenario);TestEnvFileName=$(TestEnvFilePath);TargetsWindows=$(TargetsWindows)</Properties>
      </_ProjectsToBuild>
    </ItemGroup>

    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />

    <Message Importance="High" Text="Created $(TestEnvFilePath) for scenario $(Scenario)" />
    <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" />
  </Target>

  <Target Name="CreateAllScenarioTestEnvFiles">
    <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. -->

    <ItemGroup>
      <_Scenario Include="$(Scenarios.Split(','))" />
      <_ProjectsToBuild Include="$(MSBuildProjectFile)">
        <AdditionalProperties>Scenario=%(_Scenario.Identity)</AdditionalProperties>
      </_ProjectsToBuild>
    </ItemGroup>

    <MSBuild Projects="@(_ProjectsToBuild)" Targets="CreateTestEnvFile" StopOnFirstFailure="true" />
  </Target>

  <!--
    Collect all the tasks needed to be run once, to prepare the Helix correlation payload directory.
    This just causes its dependent targets to be run.
  -->
  <Target Name="PrepareCorrelationPayloadDirectory"
          DependsOnTargets="CopyRSPFile;CreateAllScenarioTestEnvFiles" >
  </Target>

  <!--
    Parse the test asset manifest at, e.g.,
        https://dotnetfeed.blob.core.windows.net/dotnet-core/corefx-tests/4.6.0-preview6.19264.9/Linux.arm64/netcoreapp/corefx-test-assets.xml
    to figure out the tests to run. This manifest is created by the corefx official build, for use in the coreclr repo.
    
    This creates a "TestAssetBlobInfos" item group, which is used to create the Helix work items.

    NOTE: we are running this once for each Scenario, meaning we are reading and parsing it once per scenario. Ideally, we would
    do this once and pass it down to the "BuildHelixWorkItems" task later. That's a little difficult given all the re-invocations
    of this Project we do. It doesn't seem too expensive anyway.
  -->
  <Target Name="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
    <PropertyGroup>
      <_TargetGroup>netcoreapp</_TargetGroup>
      <_AssetManifestPath>$(TestAssetBlobFeedUrl)/corefx-tests/$(MicrosoftPrivateCoreFxNETCoreAppVersion)/$(__BuildOS).$(__BuildArch)/$(_TargetGroup)/corefx-test-assets.xml</_AssetManifestPath>
    </PropertyGroup>

    <ParseBuildManifest AssetManifestPath="$(_AssetManifestPath)">
      <Output TaskParameter="BlobInfos" ItemName="TestAssetBlobInfos" />
    </ParseBuildManifest>
  </Target>

  <!--
    The following item and property groups are setting things needed by Helix, to describe the jobs
    we are creating. Ideally, we would put them inside the "BuildHelixWorkItems" task, which is what
    we run to create the Helix correlation payload and work item groups. However, the Helix SDK
    references some of these in "InitialTargets", and we don't have control over the order of
    InitialTargets running, so we need to make these "global". All the "global" item and property
    groups will be evaluated before the Helix InitialTargets are run.
  -->

  <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
       setting up the per-scenario environment.
  -->

  <ItemGroup Condition=" '$(TargetsWindows)' == 'true' ">
    <HelixPreCommand Include="set __TestEnv=%HELIX_CORRELATION_PAYLOAD%\$(TestEnvFileName)" />
    <HelixPreCommand Include="type %__TestEnv%" />
    <HelixPreCommand Include="call %__TestEnv%" />
    <!-- Display the interesting COMPlus variables that are set in the environment -->
    <HelixPreCommand Include="set COMPlus" />
    <!-- Set _NT_SYMBOL_PATH so VM _ASSERTE() asserts can find the symbol files when doing stack walks -->
    <HelixPreCommand Include="set _NT_SYMBOL_PATH=%HELIX_CORRELATION_PAYLOAD%\$(TestHostPDBPath)" />
  </ItemGroup>

  <ItemGroup Condition=" '$(TargetsWindows)' != 'true' ">
    <HelixPreCommand Include="export __TestEnv=$HELIX_CORRELATION_PAYLOAD/$(TestEnvFileName)" />
    <HelixPreCommand Include="cat $__TestEnv" />
    <HelixPreCommand Include=". $__TestEnv" /> <!-- Use "." not "source"; some clients appear to run scripts with "sh" not "bash" -->
    <!-- Display the interesting COMPlus variables that are set in the environment -->
    <HelixPreCommand Include="printenv | grep COMPlus" />
  </ItemGroup>

  <PropertyGroup>
    <HelixPreCommands>@(HelixPreCommand)</HelixPreCommands>
  </PropertyGroup>

  <PropertyGroup>
    <HelixArchitecture>$(__BuildArch)</HelixArchitecture>

    <HelixConfiguration Condition=" '$(Scenario)' == 'normal' ">$(BuildType)</HelixConfiguration>
    <HelixConfiguration Condition=" '$(Scenario)' != 'normal' ">$(BuildType)-$(Scenario)</HelixConfiguration>

    <TestRunNamePrefix>CoreFX </TestRunNamePrefix>
    <TestRunNamePrefix Condition=" '$(Scenario)' == 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) @ </TestRunNamePrefix>
    <TestRunNamePrefix Condition=" '$(Scenario)' != 'normal' ">$(TestRunNamePrefix)$(BuildOS) $(BuildArch) $(BuildType) $(Scenario) @ </TestRunNamePrefix>

    <!-- REVIEW: it seems like this should be referencing "PublishTestResults" instead (without the underscore) -->
    <EnableAzurePipelinesReporter>$(_PublishTestResults)</EnableAzurePipelinesReporter>
    <EnableAzurePipelinesReporter Condition=" '$(EnableAzurePipelinesReporter)' == '' ">false</EnableAzurePipelinesReporter>

    <EnableXUnitReporter>true</EnableXUnitReporter>
    <FailOnMissionControlTestFailure>true</FailOnMissionControlTestFailure>
    <FailOnWorkItemFailure>true</FailOnWorkItemFailure>
    <WaitForWorkItemCompletion>true</WaitForWorkItemCompletion>

    <!-- Specify the retry policy for Helix jobs -->
    <!-- REVIEW: currently don't do any retries
    <MaxRetryCount Condition="'$(MaxRetryCount)' == ''">4</MaxRetryCount>
    -->
  </PropertyGroup>

  <!--
    Create all the Helix data to start a set of jobs. This is invoked once for each Scenario (see
    target "RunInParallelForEachScenario"). Create a set of work items, one for each CoreFX test assembly.
    All will have the same Scenario specific command line.  Note that this target is listed in the
    InitialTargets for this Project. This causes it to be invoked (and the Helix data created,
    such as the HelixWorkItem item group) before Helix "Test" target is invoked (as a normal target).
    We conditionalize this (and its dependent targets) with "UsesHelixSdk=true" so they aren't invoked
    during the other invocations of this Project, such as when "RunInParallelForEachScenario" is run.
  -->
  <Target Name="BuildHelixWorkItems" DependsOnTargets="GetTestAssetManifest" Condition=" '$(UsesHelixSdk)' == 'true' " >
    <Error Condition=" '$(TimeoutPerTestCollectionInMinutes)' == '' " Text="TimeoutPerTestCollectionInMinutes not set" />

    <ItemGroup Condition=" '$(UsesHelixSdk)' == 'true' ">
      <HelixCorrelationPayload Include="$(TestHostRootPath)" />

      <HelixWorkItem Include="@(TestAssetBlobInfos -> '%(FileName)')">
        <PayloadUri>$(TestAssetBlobFeedUrl)/%(Identity)</PayloadUri>
        <Command>$(HelixCommand)</Command>
        <Timeout>$([System.TimeSpan]::FromMinutes($(TimeoutPerTestCollectionInMinutes)))</Timeout>
      </HelixWorkItem>
    </ItemGroup>
  </Target>

  <Import Sdk="Microsoft.DotNet.Helix.Sdk" Project="Sdk.targets" Condition=" '$(UsesHelixSdk)' == 'true' " />

</Project>