Automate code coverage report as part of PRs. (dotnet#2194)

* Integrate coverlet for code coverage and upload reports to codecov.io
Kay-Gyasi · Jan 23, 2019 · 8e8b0c2 · 8e8b0c2
1 parent 0b50407
commit 8e8b0c2
Show file tree

Hide file tree

Showing 8 changed files with 104 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -42,6 +42,8 @@ Daily NuGet builds of the project are also available in our [MyGet](https://dotn
 
 To build ML.NET from source please visit our [developers guide](docs/project-docs/developer-guide.md).
 
+[![codecov](https://codecov.io/gh/dotnet/machinelearning/branch/master/graph/badge.svg)](https://codecov.io/gh/dotnet/machinelearning)
+
 |    | Debug | Release |
 |:---|----------------:|------------------:|
 |**Linux**|[![x64-debug](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Debug)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|[![x64-release](https://dnceng.visualstudio.com/public/_apis/build/status/dotnet/machinelearning/MachineLearning-CI?branchName=master&jobname=Linux&configuration=Build_Release)](https://dnceng.visualstudio.com/DotNet-Public/_build/latest?definitionId=104&branch=master)|

diff --git a/build/Codecoverage.proj b/build/Codecoverage.proj
@@ -0,0 +1,37 @@
+<Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Codecov">
+
+  <PropertyGroup>
+    <!-- We need to specify a framework in order for the Restore target to work -->
+    <TargetFramework>netcoreapp2.1</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+      <PackageReference Include="Codecov" Version="$(CodecovVersion)" />
+  </ItemGroup>
+  <Target Name="Codecov" DependsOnTargets="Restore">
+    <PropertyGroup>
+      <_CodecovPath>$(RestorePackagesPath)codecov\$(CodecovVersion)\tools\Codecov.exe</_CodecovPath>
+
+      <_BranchName Condition="'$(_BranchName)' == ''">$(SYSTEM_PULLREQUEST_TARGETBRANCH)</_BranchName>
+      <_BranchName Condition="'$(_BranchName)' == ''">$(BUILD_SOURCEBRANCHNAME)</_BranchName>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <_CoverageReports Include="$(BaseOutputPath)$(PlatformConfig)\coverage\*.coverage" />
+
+      <_CodecovArgs Include="-f;@(_CoverageReports)" />
+      <_CodecovArgs Include="-r;$(BUILD_REPOSITORY_NAME)" Condition="'$(BUILD_REPOSITORY_NAME)' != ''" />
+      <_CodecovArgs Include="--pr;$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)" Condition="'$(SYSTEM_PULLREQUEST_PULLREQUESTNUMBER)' != ''" />
+      <_CodecovArgs Include="-b;$(BUILD_BUILDNUMBER)" Condition="'$(BUILD_BUILDNUMBER)' != ''" />
+      <_CodecovArgs Include="--branch;$(_BranchName)" Condition="'$(_BranchName)' != ''" />
+      <_CodecovArgs Include="-c;$(BUILD_SOURCEVERSION)" Condition="'$(BUILD_SOURCEVERSION)' != ''" />
+      <_CodecovArgs Include="-n;$(BUILD_DEFINITIONNAME)" Condition="'$(BUILD_DEFINITIONNAME)' != ''" />
+      <_CodecovArgs Include="--flag;$(Configuration)" Condition="'$(Configuration)' != ''" />
+      <_CodecovArgs Include="-t;$(CodeCovToken)" Condition="'$(CodeCovToken)' != ''" />
+    </ItemGroup>
+
+    <Message Importance="high" Text="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
+    <Exec Command="&quot;$(_CodecovPath)&quot; @(_CodecovArgs, ' ')" />
+  </Target>
+
+</Project>
diff --git a/build/Dependencies.props b/build/Dependencies.props
@@ -34,6 +34,8 @@
   <!-- Build/infrastructure Dependencies -->
   <PropertyGroup>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
+    <CodecovVersion>1.1.1</CodecovVersion>
+    <CoverletVersion>2.5.1</CoverletVersion>
   </PropertyGroup>
 
   <!-- Test-only Dependencies -->

diff --git a/build/ci/phase-template.yml b/build/ci/phase-template.yml
@@ -4,13 +4,15 @@ parameters:
   buildScript: ''
   queue: {}
   customMatrixes: ''
+  codeCoverage: false
 
 phases:
   - phase: ${{ parameters.name }}
     variables:
       _buildScript: ${{ parameters.buildScript }}
       _phaseName: ${{ parameters.name }}
       _arch: ${{ parameters.architecture }}
+      _codeCoverage: ${{ parameters.codeCoverage }}
     queue:
       timeoutInMinutes: 45
       parallel: 99
@@ -35,8 +37,11 @@ phases:
     - ${{ if eq(parameters.name, 'MacOS') }}:
       - script: brew update && brew install libomp mono-libgdiplus gettext && brew link gettext --force
         displayName: Install runtime dependencies
-    - script: $(_buildScript) -$(_configuration) -runtests
-      displayName: Run Tests
+    - script: $(_buildScript) -$(_configuration) -runtests -coverage=$(_codeCoverage)
+      displayName: Run Tests.
+    - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN)
+      displayName: Upload coverage to codecov.io
+      condition: eq(variables._codeCoverage, 'true')
     - task: PublishTestResults@2
       displayName: Publish Test Results
       condition: succeededOrFailed()

diff --git a/build/codecoverage-ci.yml b/build/codecoverage-ci.yml
@@ -0,0 +1,17 @@
+################################################################################
+# ML.NET's Code Coverage validation build
+################################################################################
+
+phases:
+- template: /ci/phase-template.yml
+  parameters:
+    name: Windows_x64
+    buildScript: build.cmd
+    customMatrixes:
+      Build_Debug:
+        _configuration: Debug
+        _config_short: D
+        _includeBenchmarkData: false
+    codeCoverage: true
+    queue:
+      name: Hosted VS2017
diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,20 @@
+# https://docs.codecov.io/docs/codecov-yaml
+# https://github.com/codecov/support/wiki/Codecov-Yaml
+
+coverage:
+  status:
+    project:
+      default: false
+    patch:
+      default: false
+
+comment:
+  layout: "diff"
+
+flags:
+  production:
+    paths:
+      - src/
+  test:
+    paths:
+      - test/
diff --git a/config.json b/config.json
@@ -78,6 +78,12 @@
       "values": [],
       "defaultValue": ""
     },
+    "Coverage": {
+      "description": "Turn on code coverge.",
+      "valueType": "property",
+      "values": ["false", "true"],
+      "defaultValue": "false"
+    },
     "CleanAllProjects": {
       "description": "MsBuild target that deletes the binary output directory.",
       "valueType": "target",

diff --git a/test/Directory.Build.props b/test/Directory.Build.props
@@ -30,6 +30,19 @@
     <PackageReference Include="xunit" Version="2.4.0" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="2.4.0-prerelease-63213-02" />
+    <PackageReference Include="coverlet.msbuild" Version="$(CoverletVersion)" />
   </ItemGroup>
 
+  <PropertyGroup Condition="'$(Coverage)' == 'true'">
+      <CollectCoverage>true</CollectCoverage>
+      <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies> <!-- https://github.com/tonerdo/coverlet/issues/72 -->
+      <CoverletOutputFormat>opencover</CoverletOutputFormat>
+      <CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\$(MSBuildProjectName).coverage</CoverletOutput>
+      <Include>[Microsoft.ML.*]*</Include>
+      <!-- Excluding for perf reasons. These classes have tests that can be run conditionally
+           but they need to be migrated. Excluding these classes should have very minimal effect on code coverage.
+      -->
+      <Exclude>[*]Microsoft.ML.*Contracts*,[*]Microsoft.ML.Internal.Utilities*,[*]Microsoft.ML.Data.VBuffer*</Exclude>
+  </PropertyGroup>
+
 </Project>