Enable dotnet test in libraries (dotnet#35285)

* Enable dotnet test * Update docs * Use vstest in F5 scenarios * Make dotnet test without framework switch work * Code cleanup * Only run code coverage conditionally * Fix F5 condition for netcoreapp * Downgrade sdk to check for helix submission failures * Fix wrong conditions * Add blame data collector
YesReddy1 · May 4, 2020 · 200b197 · 200b197
1 parent a1af15d
commit 200b197
Show file tree

Hide file tree

Showing 28 changed files with 254 additions and 292 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -2,12 +2,6 @@
   "version": 1,
   "isRoot": true,
   "tools": {
-    "coverlet.console": {
-      "version": "1.7.0",
-      "commands": [
-        "coverlet"
-      ]
-    },
     "dotnet-reportgenerator-globaltool": {
       "version": "4.5.2",
       "commands": [

diff --git a/Directory.Build.targets b/Directory.Build.targets
@@ -26,4 +26,5 @@
     <LangVersion>preview</LangVersion>
     <LangVersion Condition="'$(Language)' == 'VB'">latest</LangVersion>
   </PropertyGroup>
+
 </Project>
diff --git a/docs/workflow/building/libraries/code-coverage.md b/docs/workflow/building/libraries/code-coverage.md
@@ -28,40 +28,30 @@ An issue need not be addressed in its entirety. We happily accept contributions
 
 You can perform code coverage runs for the entire repository locally by adding the `coverage` switch (assuming that  source and test assemblies are already built):
 
-    build -test -coverage
+    build libs.tests -test -coverage
 
 This runs the tests and generates the full code coverage report. The resulting index.htm file providing the results of the run should be available at:
 
     artifacts\coverage\index.htm
 
-You can also build and test with code coverage for a particular test project rather than for the whole repo with the ```/p:Coverage=true``` argument:
+You can also build and test with code coverage for a particular test project rather than for the whole repo with the `/p:Coverage=true` property:
 
-    dotnet build /t:Test /p:Coverage=true
+    dotnet test -f netcoreapp5.0 /p:Coverage=true
 
 The results for this one library will then be available in this index.htm file, where $(OutDir) is the directory where the binaries were generated.
 
     $(OutDir)\report\index.htm
 
 For example, to build, test, and get code coverage results for the System.Diagnostics.Debug library, from the root of the repo one can do:
 
-    cd src\System.Diagnostics.Debug\tests\
-    dotnet build /t:Test /p:Coverage=true
+    dotnet test src\libraries\System.Diagnostics.Debug -f netcoreapp5.0 /p:Coverage=true
 
 And then once the run completes:
 
     $(OutDir)\report\index.htm
 
-**Note:** If you only want to measure the coverage of your local changes (that haven't been pushed to git), run:
-
-    dotnet build /t:Test /p:Coverage=true /p:CoverageSourceLink=false
-
-
 ## Code coverage with System.Private.CoreLib code
 
-Some of the libraries for which contracts and tests live in the corefx repo are actually fully or partially implemented in the core runtime library in another repo, e.g. the implementation that backs the System.Runtime contract is in System.Private.CoreLib.dll in either the coreclr or corert repo. Test projects for code that lives, fully or partially, in System.Private.CoreLib, should have the property `TestRuntime` set to `true` in order to obtain proper code coverage reports.
+Some of the libraries for which contracts and tests live in libraries are actually fully or partially implemented in the core runtime library, e.g. the implementation that backs the System.Runtime contract is in System.Private.CoreLib.dll. Test projects for code that lives, fully or partially, in System.Private.CoreLib, should have the property `TestRuntime` set to `true` in order to obtain proper code coverage reports.
 
 If the test project does not set the property `TestRuntime` to `true` and you want to collect code coverage that includes types in System.Private.CoreLib.dll add `/p:TestRuntime=true` to the coverage build command listed above.
-
-If you want to get coverage report against a private build of System.Private.CoreLib (// TODO //).
-
-The build and test projects take care of copying assemblies and PDBs as needed for coverage runs. The resulting code coverage report should now also include details for System.Private.CoreLib.
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -116,6 +116,7 @@
     <MicrosoftNETTestSdkVersion>16.7.0-preview-20200429-01</MicrosoftNETTestSdkVersion>
     <MicrosoftDotNetXHarnessTestsRunnersVersion>1.0.0-prerelease.20230.6</MicrosoftDotNetXHarnessTestsRunnersVersion>
     <XUnitVersion>2.4.1</XUnitVersion>
+    <CoverletCollectorVersion>1.2.0</CoverletCollectorVersion>
     <TraceEventVersion>2.0.5</TraceEventVersion>
     <NewtonsoftJsonVersion>12.0.3</NewtonsoftJsonVersion>
     <MoqVersion>4.12.0</MoqVersion>

diff --git a/eng/referenceFromRuntime.targets b/eng/referenceFromRuntime.targets
@@ -19,7 +19,7 @@
   </PropertyGroup>
 
   <Target Name="AddRuntimeProjectReference"
-          Condition="'$(IsTestProject)'!='true' and '@(ReferenceFromRuntime)' != ''">
+          Condition="'$(IsTestProject)' != 'true' and '$(IsTestSupportProject)' != 'true' and '@(ReferenceFromRuntime)' != ''">
     <Error Text="ReferenceFromRuntime may not be used from reference assemblies."
            Condition="'$(IsReferenceAssembly)' == 'true' and '$(AllowReferenceFromRuntime)' != 'true'" />
 
@@ -65,7 +65,7 @@
       <_referencePathFromRuntime Include="@(RuntimeFiles)" Private="false" />
       <_referencePathFromRuntime Include="@(_referencePathFromRestoredRuntime)" Private="false" />
       <!-- If this is a test project, also use the $(RuntimePath) to find a @(ReferenceFromRuntime) assembly. -->
-      <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true'" />
+      <_referencePathFromRuntime Include="@(ReferenceFromRuntime->'$(RuntimePath)%(Identity).dll')" Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true'" />
       <!-- transform to filename in order to intersect -->
       <_referencePathFromRuntimeByFileName Include="@(_referencePathFromRuntime->'%(FileName)')" Condition="'%(_referencePathFromRuntime.Extension)' == '.dll'" >
         <ReferencePath>%(Identity)</ReferencePath>

diff --git a/eng/references.targets b/eng/references.targets
@@ -36,7 +36,7 @@
     </ItemGroup>
   </Target>
 
-  <Target Name="AddDefaultTestReferences" BeforeTargets="SetupDefaultReferences" Condition="'$(IsTestProject)' == 'true'">
+  <Target Name="AddDefaultTestReferences" BeforeTargets="SetupDefaultReferences" Condition="'$(IsTestProject)' == 'true' or '$(IsTestSupportProject)' == 'true'">
     <ItemGroup>
       <DefaultReferenceExclusions Include="@(ReferenceFromRuntime)"/>
 

diff --git a/eng/testing/.runsettings b/eng/testing/.runsettings
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="utf-8"?>
+<RunSettings>
+  <RunConfiguration>
+    <!-- Timeout in ms, 5 minutes -->
+    <TestSessionTimeout>300000</TestSessionTimeout>
+    <!-- Directory for test run reports. E.g. trx, coverage etc. -->
+    <ResultsDirectory>.\</ResultsDirectory>
+    <!-- Working directory for test invocation. Results directory can be relative to this. Used by IDEs. -->
+    <SolutionDirectory>.\</SolutionDirectory>
+    <!-- Degree of parallelization, spawns n test hosts to run tests. -->
+    <MaxCpuCount>$$MAXCPUCOUNT$$</MaxCpuCount>
+    <!-- Configures the architecture of test host. -->
+    <TargetPlatform>$$TARGETPLATFORM$$</TargetPlatform>
+    <!-- If true, an adapter should disable any test case parallelization. -->
+    <DisableParallelization>$$DISABLEPARALLELIZATION$$</DisableParallelization>
+    <!-- If true, an adapter shouldn't create appdomains to run tests. -->
+    <DisableAppDomain>$$DISABLEAPPDOMAIN$$</DisableAppDomain>
+    <!-- Filter out failing (wrong framwork, platform, runtime or activeissue) tests -->
+    <TestCaseFilter>$$TESTCASEFILTER$$</TestCaseFilter>
+    <DotNetHostPath>$$DOTNETHOSTPATH$$</DotNetHostPath>
+    <EnvironmentVariables>
+      <!-- Use our self-built framework on .NET Framework -->
+      <DEVPATH>$$DEVPATH$$</DEVPATH>
+    </EnvironmentVariables>
+  </RunConfiguration>
+  <LoggerRunSettings>
+    <Loggers>
+      <Logger uri="logger://Microsoft/TestPlatform/TrxLogger/v1">
+        <Configuration>
+          <LogFileName>testResults.trx</LogFileName>
+        </Configuration>
+      </Logger>
+      <Logger uri="logger://Microsoft/TestPlatform/HtmlLogger/v1">
+        <Configuration>
+          <LogFileName>testResults.html</LogFileName>
+        </Configuration>
+      </Logger>
+      <Logger friendlyName="console">
+        <Configuration>
+          <Verbosity>Minimal</Verbosity>
+        </Configuration>
+      </Logger>
+    </Loggers>
+  </LoggerRunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="XPlat code coverage" enabled="$$COVERAGE_ENABLED$$">
+        <Configuration>
+          <Include>$$COVERAGE_INCLUDE$$</Include>
+          <ExcludeByFile>$$COVERAGE_EXCLUDEBYFILE$$</ExcludeByFile>
+          <IncludeDirectory>$$COVERAGE_INCLUDEDIRECTORY$$</IncludeDirectory>
+          <Format>opencover</Format>
+          <SingleHit>false</SingleHit>
+          <UseSourceLink>true</UseSourceLink>
+          <IncludeTestAssembly>false</IncludeTestAssembly>
+        </Configuration>
+      </DataCollector>
+      <DataCollector friendlyName="blame" enabled="true">
+        <Configuration>
+          <CollectDump CollectAlways="false" DumpType="mini" />
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>
diff --git a/eng/testing/coverage.props b/eng/testing/coverage.props
diff --git a/eng/testing/coverage.targets b/eng/testing/coverage.targets
@@ -1,39 +1,5 @@
-<Project InitialTargets="SetupCoverageFilter">
-  <!-- Wrap RunCommand and RunArguments inside code coverage invocation. -->
-  <PropertyGroup>
-    <RunArguments>"$(TargetFileName)" --target "$(RunCommand)" --targetargs "$(RunArguments)" --format "$(CoverageFormat)" --output "$(CoverageOutputPath)" --threshold "$(CoverageThreshold)" --threshold-type "$(CoverageThresholdType)" --verbosity "$(CoverageVerbosity)"</RunArguments>
-    <RunArguments Condition="'$(CoverageSourceLink)' == 'true'">$(RunArguments) --use-source-link</RunArguments>
-    <RunCommand>"$(DotNetTool)" tool run coverlet</RunCommand>
-  </PropertyGroup>
-
-  <!-- Coverage report -->
-  <PropertyGroup>
-    <CoverageReportDir Condition="'$(CoverageReportDir)' == ''">$([MSBuild]::NormalizeDirectory('$(OutDir)', 'report'))</CoverageReportDir>
-    <CoverageReportResultsPath>$([MSBuild]::NormalizePath('$(CoverageReportDir)', 'index.htm'))</CoverageReportResultsPath>
-    <CoverageReportCommandLine>"$(DotNetTool)" tool run reportgenerator "-reports:$(CoverageReportInputPath)" "-targetdir:$(CoverageReportDir.TrimEnd('\/'))" "-reporttypes:$(CoverageReportTypes)" "-verbosity:$(CoverageReportVerbosity)"</CoverageReportCommandLine>
-  </PropertyGroup>
-
-  <!-- Skip generating individual reports if a full report is generated. -->
-  <ItemGroup Condition="'$(BuildAllProjects)' != 'true' and '$(SkipCoverageReport)' != 'true'">
-    <PostRunScriptCommands Include="$(CoverageReportCommandLine)" />
-  </ItemGroup>
-
+<Project>
   <Target Name="SetupCoverageFilter">
-    <PropertyGroup Condition="'@(CoverageExcludeFile)' != ''">
-      <CoverageExcludeByFileFilter>--exclude-by-file @(CoverageExcludeFile -> '"%(Identity)"', ' --exclude-by-file ')</CoverageExcludeByFileFilter>
-      <RunArguments>$(RunArguments) $(CoverageExcludeByFileFilter)</RunArguments>
-    </PropertyGroup>
-
-    <PropertyGroup Condition="'@(CoverageProbePath)' != ''">
-      <IncludeDirectoriesFilter>--include-directory @(CoverageProbePath -> '"$(RunScriptHostDir)%(Identity)"', ' --include-directory ')</IncludeDirectoriesFilter>
-      <RunArguments>$(RunArguments) $(IncludeDirectoriesFilter)</RunArguments>
-    </PropertyGroup>
-
-    <PropertyGroup Condition="'@(CoverageExclude)' != ''">
-      <CoverageExcludeFilter>--exclude @(CoverageExclude -> '"%(Identity)"', ' --exclude ')</CoverageExcludeFilter>
-      <RunArguments>$(RunArguments) $(CoverageExcludeFilter)</RunArguments>
-    </PropertyGroup>
-
     <!--
       We need to filter the data to only the assembly being tested. Otherwise we will gather tons of data about other assemblies.
       If the code being tested is part of the runtime itself, it requires special treatment.
@@ -48,15 +14,40 @@
       CoverageAssemblies can be passed in to the build to gather coverage on additional assemblies.
     -->
     <ItemGroup>
-      <_CoverageAssemblies Include="$(AssemblyBeingTested)" />
-      <_CoverageAssemblies Include="System.Private.CoreLib" Condition="'$(TestRuntime)' == 'true'" />
-      <_CoverageAssemblies Include="@(AssembliesBeingTested)" />
-      <_CoverageAssemblies Include="$(CoverageAssemblies)" Condition="'$(CoverageAssemblies)' != ''" />
+      <CoverageInclude Include="$(AssemblyBeingTested)" />
+      <CoverageInclude Include="System.Private.CoreLib" Condition="'$(TestRuntime)' == 'true'" />
+      <CoverageInclude Include="@(AssembliesBeingTested)" />
+      <CoverageInclude Include="$(CoverageAssemblies)" Condition="'$(CoverageAssemblies)' != ''" />
     </ItemGroup>
 
-    <PropertyGroup Condition="'$(CoverageType)' != 'all'">
-      <CoverageFilter>--include @(_CoverageAssemblies -> '"[%(Identity)]*"', ' --include ')</CoverageFilter>
-      <RunArguments>$(RunArguments) $(CoverageFilter)</RunArguments>
+    <PropertyGroup>
+      <CoverageIncludeFilter>@(CoverageInclude -> '[%(Identity)]*', ',')</CoverageIncludeFilter>
     </PropertyGroup>
+
+    <PropertyGroup Condition="'@(CoverageExcludeByFile)' != ''">
+      <CoverageExcludeByFileFilter>@(CoverageExcludeByFile -> '%(Identity)', ',')</CoverageExcludeByFileFilter>
+    </PropertyGroup>
+
+    <PropertyGroup Condition="'@(CoverageIncludeDirectory)' != ''">
+      <CoverageIncludeDirectoryFilter>@(CoverageIncludeDirectory -> '$(TestHostRootPath)%(Identity)', ',')</CoverageIncludeDirectoryFilter>
+    </PropertyGroup>
+  </Target>
+
+  <Target Name="GenerateCoverageReport"
+          Condition="'$(Coverage)' == 'true' and '$(SkipCoverageReport)' != 'true'"
+          AfterTargets="VSTest">
+    <ItemGroup Condition="'$(CoverageReportInputPath)' == ''">
+      <CoverageOutputFile Include="$(OutDir)*\coverage.opencover.xml" />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <CoverageReportInputPath Condition="'$(CoverageReportInputPath)' == ''">%(CoverageOutputFile.Identity)</CoverageReportInputPath>
+      <CoverageReportTypes Condition="'$(CoverageReportTypes)' == ''">Html</CoverageReportTypes>
+      <CoverageReportVerbosity Condition="'$(CoverageReportVerbosity)' == ''">Info</CoverageReportVerbosity>
+      <CoverageReportDir Condition="'$(CoverageReportDir)' == ''">$([MSBuild]::NormalizeDirectory('$(OutDir)', 'report'))</CoverageReportDir>
+      <CoverageReportCommand>"$(DotNetTool)" tool run reportgenerator "-reports:$(CoverageReportInputPath)" "-targetdir:$(CoverageReportDir.TrimEnd('\/'))" "-reporttypes:$(CoverageReportTypes)" "-verbosity:$(CoverageReportVerbosity)"</CoverageReportCommand>
+    </PropertyGroup>
+
+    <Exec Command="$(CoverageReportCommand)" />
   </Target>
 </Project>
diff --git a/eng/testing/launchSettings.json b/eng/testing/launchSettings.json
diff --git a/eng/testing/launchSettings.targets b/eng/testing/launchSettings.targets