Enable restore for ref and src projects in libraries (dotnet#33242)

* Enable restore for projects in libraries Enabling project restore for ref and src projects * Remove winrt.depproj * Remove tools.depproj * Enable upfront restore * Fix casing of PInvokeAnalyzerException file * Remove harvestPackages.depproj and refactoring Removing havestPackages.depproj in favor of a target that is hooked into Arcade's repo restore phase in Tools.props. Additionally moving common properties that are used both in Arcade and the repo inside Configurations.props. Before they were scattered around multiple places (Versions.props, Subsets.props). * Set ilasm/ildasm RID for restore * Sequence restore into steps out of upfront restore * Fix docker entrypoints by using build.cmd/sh
garora · Mar 10, 2020 · 981cc64 · 981cc64
1 parent 4d2a2da
commit 981cc64
Show file tree

Hide file tree

Showing 39 changed files with 284 additions and 332 deletions.
diff --git a/Directory.Build.props b/Directory.Build.props
@@ -88,6 +88,9 @@
 
     <!-- Resource naming bug: https://github.com/microsoft/msbuild/issues/4740 -->
     <EmbeddedResourceUseDependentUponConvention>false</EmbeddedResourceUseDependentUponConvention>
+
+    <!-- Enable NuGet static graph evaluation to optimize incremental restore: https://github.com/dotnet/sdk/pull/10794 -->
+    <!-- <RestoreUseStaticGraphEvaluation>true</RestoreUseStaticGraphEvaluation> -->
   </PropertyGroup>
 
   <PropertyGroup>

diff --git a/docs/coding-guidelines/project-guidelines.md b/docs/coding-guidelines/project-guidelines.md
@@ -10,9 +10,9 @@ once before you can iterate and work on a given library project.
  - Netstandard Library - Copy to `bin\ref\netstandard2.0`
  - NetFx targeting pack - Copy to `bin\ref\net472`
 - Build targeting pack
- - Build src\ref.builds which builds all references assembly projects. For reference assembly project information see [ref](#ref)
+ - Build src\libraries\ref.proj which builds all references assembly projects. For reference assembly project information see [ref](#ref)
 - Build product
- - Build src\src.builds which builds all the source library projects. For source library project information see [src](#src).
+ - Build src\libraries\src.proj which builds all the source library projects. For source library project information see [src](#src).
 - Sign product
  - Build src\sign.proj
 

diff --git a/eng/Analyzers.props b/eng/Analyzers.props
@@ -1,6 +1,8 @@
 <Project>
   <PropertyGroup>
     <CodeAnalysisRuleset>$(MSBuildThisFileDirectory)CodeAnalysis.ruleset</CodeAnalysisRuleset>
+    <!-- Disable analyzers in sourcebuild -->
+    <EnableAnalyzers Condition="'$(DotNetBuildFromSource)' == 'true'">false</EnableAnalyzers>
   </PropertyGroup>
   <ItemGroup Condition="'$(EnableAnalyzers)' == 'true'">
     <PackageReference Include="Microsoft.DotNet.CodeAnalysis" Version="$(MicrosoftDotNetCodeAnalysisVersion)" PrivateAssets="all" IsImplicitlyDefined="true" />

diff --git a/eng/Build.props b/eng/Build.props
@@ -15,6 +15,7 @@
       https://github.com/dotnet/arcade/issues/388
   -->
 
+  <!-- Import to have access to properties/items used both during the Arcade phase and in the repo. -->
   <Import Project="$(MSBuildThisFileDirectory)Configurations.props" />
   <Import Project="$(MSBuildThisFileDirectory)Subsets.props" />
 
@@ -76,8 +77,12 @@
     </ItemGroup>
 
     <MSBuild Projects="@(RepoTaskProjects)"
-             Properties="Configuration=Debug;Platform=AnyCPU"
-             Targets="Restore;Build"/>
+             Properties="Configuration=Debug;Platform=AnyCPU;__BuildPhase=Restore"
+             Targets="Restore" />
+
+    <MSBuild Projects="@(RepoTaskProjects)"
+             Properties="Configuration=Debug;Platform=AnyCPU;__BuildPhase=Build"
+             Targets="Build" />
 
     <WriteLinesToFile File="$(RepoTasksOutputFile)"
                       Lines="$(RepoTasksOutputFile)"

diff --git a/eng/Configurations.props b/eng/Configurations.props
@@ -1,4 +1,24 @@
 <Project>
+  <!--
+    This file contains properties and items which are used in both the Arcade
+    steps and in the repository.
+  -->
+
+  <PropertyGroup>
+    <LibrariesProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'libraries'))</LibrariesProjectRoot>
+    <CoreClrProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr'))</CoreClrProjectRoot>
+    <MonoProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono'))</MonoProjectRoot>
+    <InstallerProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'installer'))</InstallerProjectRoot>
+    <RepoToolsLocalDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'tools-local'))</RepoToolsLocalDir>
+    <RepoTasksDir>$([MSBuild]::NormalizeDirectory('$(RepoToolsLocalDir)', 'tasks'))</RepoTasksDir>
+    <IbcOptimizationDataDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'ibc'))</IbcOptimizationDataDir>
+    <XmlDocDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'docs'))</XmlDocDir>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <EnableNgenOptimization Condition="'$(EnableNgenOptimization)' == '' and '$(Configuration)' == 'Release'">true</EnableNgenOptimization>
+  </PropertyGroup>
+
   <!-- Honor the generic RuntimeConfiguration property. -->
   <PropertyGroup>
     <RuntimeConfiguration Condition="'$(RuntimeConfiguration)' == ''">$(Configuration)</RuntimeConfiguration>

diff --git a/eng/Subsets.props b/eng/Subsets.props
@@ -47,15 +47,6 @@
       artifacts to the test layout, then running the test subset.
   -->
 
-  <PropertyGroup>
-    <LibrariesProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'libraries'))</LibrariesProjectRoot>
-    <CoreClrProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'coreclr'))</CoreClrProjectRoot>
-    <MonoProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'mono'))</MonoProjectRoot>
-    <InstallerProjectRoot>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'installer'))</InstallerProjectRoot>
-    <RepoToolsLocalDir>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'tools-local'))</RepoToolsLocalDir>
-    <RepoTasksDir>$([MSBuild]::NormalizeDirectory('$(RepoToolsLocalDir)', 'tasks'))</RepoTasksDir>
-  </PropertyGroup>
-
   <PropertyGroup>
     <DefaultSubsetCategories>libraries-installer-coreclr-mono</DefaultSubsetCategories>
     <DefaultInstallerSubsets>corehost-managed-depproj-pkgproj-bundle-installers-test</DefaultInstallerSubsets>

diff --git a/eng/Tools.props b/eng/Tools.props
@@ -1,39 +1,13 @@
 <Project>
-  <PropertyGroup>
-    <MSBuildTreatWarningsAsErrors>false</MSBuildTreatWarningsAsErrors>
-    <EnableAnalyzers>true</EnableAnalyzers>
-  </PropertyGroup>
+  <!-- Import to have access to properties/items used both during the Arcade phase and in the repo. -->
+  <Import Project="$(MSBuildThisFileDirectory)Configurations.props" />
 
-  <!-- We need to import this props file which contains PackageReferences to analyzers so that
-       analyzer build assets are imported via the Tools generated props and targets -->
-  <Import Project="$(RepositoryEngineeringDir)Analyzers.props" Condition="Exists('$(RepositoryEngineeringDir)Analyzers.props') and '$(DotNetBuildFromSource)' != 'true'" />
-
-  <!-- source-built packages -->
   <ItemGroup>
-    <!-- arcade -->
-    <PackageReference Include="Microsoft.DotNet.GenFacades" Version="$(MicrosoftDotNetGenFacadesVersion)" />
     <PackageReference Include="Microsoft.DotNet.Build.Tasks.Packaging" Version="$(MicrosoftDotNetBuildTasksPackagingVersion)" />
-
-    <!-- coreclr -->
-    <!-- Download the package in the initial arcade restore step to work around race conditions when restoring an msbuild SDK. -->
-    <PackageDownload Include="Microsoft.NET.Sdk.IL" Version="[$(MicrosoftNETSdkILVersion)]" />
-
-    <!-- roslyn -->
-    <PackageReference Include="Microsoft.Net.Compilers.Toolset" Version="$(MicrosoftNetCompilersToolsetVersion)" PrivateAssets="all" IsImplicitlyDefined="true" />
-
-    <!-- mono -->
-    <PackageReference Include="illink.tasks" Version="$(ILLinkTasksVersion)" PrivateAssets="all" IsImplicitlyDefined="true" ExcludeAssets="build" GeneratePathProperty="true" />
-  </ItemGroup>
-
-  <!-- excluded from offline portion of source build -->
-  <ItemGroup Condition="'$(DotNetBuildOffline)' != 'true'">
-    <!-- arcade -->
-    <PackageReference Include="Microsoft.DotNet.GenAPI" Version="$(MicrosoftDotNetGenApiVersion)" />
   </ItemGroup>
 
   <!-- excluded from source build -->
   <ItemGroup Condition="'$(DotNetBuildFromSource)' != 'true'">
-    <PackageReference Include="Microsoft.DotNet.ApiCompat" Version="$(MicrosoftDotNetApiCompatVersion)" />
     <PackageReference Include="Microsoft.DotNet.Build.Tasks.Feed" Version="$(MicrosoftDotNetBuildTasksFeedVersion)" />
     <PackageReference Include="Microsoft.DotNet.VersionTools.Tasks" Version="$(MicrosoftDotNetVersionToolsTasksVersion)" />
 

diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -38,13 +38,13 @@
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>421b20fb860907c45673de791177e3fd61402197</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.CodeAnalysis" Version="5.0.0-beta.20153.1">
+    <Dependency Name="Microsoft.DotNet.CodeAnalysis" Version="5.0.0-beta.20156.4">
       <Uri>https://github.com/dotnet/arcade</Uri>
       <Sha>421b20fb860907c45673de791177e3fd61402197</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk" Version="5.0.0-beta.20153.1">
+    <Dependency Name="Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk" Version="5.0.0-beta.20155.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>421b20fb860907c45673de791177e3fd61402197</Sha>
+      <Sha>bc4fa8e7149769db4efd466f160417a32b11f0bf</Sha>
     </Dependency>
     <Dependency Name="Microsoft.DotNet.RemoteExecutor" Version="5.0.0-beta.20153.1">
       <Uri>https://github.com/dotnet/arcade</Uri>

diff --git a/eng/Versions.props b/eng/Versions.props
@@ -19,10 +19,6 @@
     <UsingToolMicrosoftNetCompilers>true</UsingToolMicrosoftNetCompilers>
     <UsingToolIbcOptimization>true</UsingToolIbcOptimization>
     <UsingToolXliff>false</UsingToolXliff>
-    <!-- Paths used during restore -->
-    <EnableNgenOptimization Condition="'$(EnableNgenOptimization)' == '' and '$(Configuration)' == 'Release'">true</EnableNgenOptimization>
-    <IbcOptimizationDataDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'ibc'))</IbcOptimizationDataDir>
-    <XmlDocDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsBinDir)', 'docs'))</XmlDocDir>
     <!-- Blob storage container that has the "Latest" channel to publish to. -->
     <ContainerName>dotnet</ContainerName>
     <ChecksumContainerName>$(ContainerName)</ChecksumContainerName>
@@ -60,7 +56,7 @@
     <!-- Arcade dependencies -->
     <MicrosoftDotNetApiCompatVersion>5.0.0-beta.20153.1</MicrosoftDotNetApiCompatVersion>
     <MicrosoftDotNetBuildTasksFeedVersion>5.0.0-beta.20153.1</MicrosoftDotNetBuildTasksFeedVersion>
-    <MicrosoftDotNetCodeAnalysisVersion>5.0.0-beta.20153.1</MicrosoftDotNetCodeAnalysisVersion>
+    <MicrosoftDotNetCodeAnalysisVersion>5.0.0-beta.20156.4</MicrosoftDotNetCodeAnalysisVersion>
     <MicrosoftDotNetGenAPIVersion>5.0.0-beta.20153.1</MicrosoftDotNetGenAPIVersion>
     <MicrosoftDotNetGenFacadesVersion>5.0.0-beta.20153.1</MicrosoftDotNetGenFacadesVersion>
     <MicrosoftDotNetXUnitExtensionsVersion>5.0.0-beta.20153.1</MicrosoftDotNetXUnitExtensionsVersion>

diff --git a/eng/codeAnalysis.targets b/eng/codeAnalysis.targets
diff --git a/eng/docker/build-docker-sdk.ps1 b/eng/docker/build-docker-sdk.ps1
@@ -18,8 +18,7 @@ if ($buildWindowsContainers)
   # Due to size concerns, we don't currently do docker builds on windows.
   # Build on the host machine, then simply copy artifacts to the target docker image.
   # This should result in significantly lower build times, for now.
-  & "$REPO_ROOT_DIR/coreclr.cmd" -c Release
-  & "$REPO_ROOT_DIR/libraries.cmd" -ci -c $configuration -runtimeConfiguration release
+  & "$REPO_ROOT_DIR/build.cmd" -ci -subsetcategory coreclr-libraries -runtimeconfiguration release -c $configuration
 
   # Dockerize the build artifacts
   docker build --tag $imageName `

diff --git a/eng/docker/libraries-sdk.linux.Dockerfile b/eng/docker/libraries-sdk.linux.Dockerfile
@@ -8,8 +8,7 @@ WORKDIR /repo
 COPY . .
 
 ARG CONFIGURATION=Release
-RUN ./src/coreclr/build.sh -release -skiptests -clang9 && \
-    ./libraries.sh -c $CONFIGURATION -runtimeconfiguration release
+RUN ./build.sh -ci -subsetcategory coreclr-libraries -runtimeconfiguration release -c $CONFIGURATION
 
 FROM $SDK_BASE_IMAGE as target
 

diff --git a/eng/pipelines/libraries/enterprise/linux.yml b/eng/pipelines/libraries/enterprise/linux.yml
@@ -26,7 +26,7 @@ variables:
   - name: enterpriseTestsSetup
     value: $(sourcesRoot)/Common/tests/System/Net/EnterpriseTests/setup
   - name: containerRunTestsCommand
-    value: /repo/.dotnet/dotnet build /t:test --no-restore
+    value: /repo/.dotnet/dotnet build /t:test
   - name: containerLibrariesRoot
     value: /repo/src/libraries
 
@@ -50,7 +50,7 @@ steps:
   displayName: Test linuxclient connection to web server
 
 - bash: |
-    docker exec linuxclient bash -c '/repo/src/coreclr/build.sh -release -skipnuget -clang9 && /repo/libraries.sh /p:CoreCLRConfiguration=Release'
+    docker exec linuxclient bash -c '/repo/build.sh -subsetcategory coreclr-libraries -runtimeconfiguration release -ci'
   displayName: Build product sources
 
 - bash: |

diff --git a/...e/harvestPackages/harvestPackages.depproj → eng/restore/harvestPackages.targets b/...e/harvestPackages/harvestPackages.depproj → eng/restore/harvestPackages.targets
@@ -1,15 +1,14 @@
-<Project InitialTargets="AddPackageDownload" Sdk="Microsoft.NET.Sdk">
+<Project InitialTargets="AddPackageDownload">
   <PropertyGroup>
-    <TargetFrameworks>netstandard2.0</TargetFrameworks>
+    <PackagingTaskAssembly>$(NuGetPackageRoot)microsoft.dotnet.build.tasks.packaging\$(MicrosoftDotNetBuildTasksPackagingVersion)\tools\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' == 'core'">$(PackagingTaskAssembly)netcoreapp2.1\</PackagingTaskAssembly>
+    <PackagingTaskAssembly Condition="'$(MSBuildRuntimeType)' != 'core'">$(PackagingTaskAssembly)net472\</PackagingTaskAssembly>
+    <PackagingTaskAssembly>$(PackagingTaskAssembly)Microsoft.DotNet.Build.Tasks.Packaging.dll</PackagingTaskAssembly>
   </PropertyGroup>
 
-  <Import Project="$(PkgDir)baseline/baseline.props" />
-
-  <!-- only restore this project during the build, don't copy any of it's packages
-       The sole purpose of this project is to download packages that can be examined
-       for harvesting binaries & support. -->
-  <Target Name="Build" DependsOnTargets="RestorePackages" />
+  <Import Project="$(LibrariesProjectRoot)pkg\baseline\baseline.props" />
 
+  <UsingTask TaskName="GetLastStablePackage" AssemblyFile="$(PackagingTaskAssembly)"/>
   <Target Name="AddPackageDownload">
     <ItemGroup>
       <_AllPkgProjs Include="$(LibrariesProjectRoot)*\pkg\**\*.pkgproj" />
@@ -20,10 +19,10 @@
     </ItemGroup>
 
     <GetLastStablePackage
-        LatestPackages="@(_AllPkgProjsToPackageIdentity)"
-        PackageIndexes="$(PackageIndexFile)"
-        DoNotAllowVersionsFromSameRelease="true">
-        <Output TaskParameter="LastStablePackages" ItemName="_PackageDownload" />
+      LatestPackages="@(_AllPkgProjsToPackageIdentity)"
+      PackageIndexes="$(PackageIndexFile)"
+      DoNotAllowVersionsFromSameRelease="true">
+      <Output TaskParameter="LastStablePackages" ItemName="_PackageDownload" />
     </GetLastStablePackage>
 
     <!-- Allow to override package download and versions in case there is already a PackageDownload set -->
@@ -40,5 +39,4 @@
       </PackageDownload>
     </ItemGroup>
   </Target>
-
 </Project>
diff --git a/eng/restore/repoRestore.props b/eng/restore/repoRestore.props
@@ -28,4 +28,4 @@
     <ComputeNETCoreBuildOutputFiles>false</ComputeNETCoreBuildOutputFiles>
     <GenerateRuntimeConfigurationFiles>false</GenerateRuntimeConfigurationFiles>
   </PropertyGroup>
-</Project>
+</Project>
diff --git a/eng/restore/repoRestore.targets b/eng/restore/repoRestore.targets
@@ -4,8 +4,6 @@
     <RestoreProjectStyle Condition="'$(MSBuildProjectExtension)' != '.depproj'">Unknown</RestoreProjectStyle>
   </PropertyGroup>
 
-  <Import Project="$(RepositoryEngineeringDir)codeAnalysis.targets" />
-
   <PropertyGroup>
     <!-- Nuget will normally disable all generated imports as part of a restore to avoid 
          non-determinism where first restore changes a subsequent restore,

diff --git a/global.json b/global.json
@@ -12,7 +12,7 @@
     "python": "2.7.15"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "5.0.0-beta.20153.1",
+    "Microsoft.DotNet.Build.Tasks.TargetFramework.Sdk": "5.0.0-beta.20156.4",
     "Microsoft.DotNet.Arcade.Sdk": "5.0.0-beta.20153.1",
     "Microsoft.DotNet.Build.Tasks.SharedFramework.Sdk": "5.0.0-beta.20153.1",
     "Microsoft.DotNet.Helix.Sdk": "5.0.0-beta.20153.1",