Change how we build the cross-OS DAC to support building in the VMR (d…

…otnet#111927)

* Don't build the linuxdac by default. Cross-os DAC build failures are quite rare nowadays and we have CI protection against build failures. Remove the linux dac from the default build subsets to speed up the default build experience.
* Rewrite cross-os DAC packaging in terms of runtime packs instead of using a separate artifact
* Build the cross-os DACs when crossdacpack is built.
* Forward the ESRP tool path down from DotNetBuild.props and set up the cross-dac invocation from the join jobs.
* Pass down the buildpass to the inner build and decide what to build in Subsets.props
* Allow passing a script root (needed for VMR reusage of this template)
* Adjust DAC signing script to fix failures in internal testing
* Force the value of SignRequestFiles to always be an array
* Build the cross-os DACs via their subset names

eng/DotNetBuild.props

@@ -72,6 +72,7 @@
       <InnerBuildArgs Condition="'$(DotNetBuildMonoAOTEnableLLVM)' != ''">$(InnerBuildArgs) /p:MonoAOTEnableLLVM=$(DotNetBuildMonoAOTEnableLLVM)</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(DotNetBuildMonoBundleLLVMOptimizer)' != ''">$(InnerBuildArgs) /p:MonoBundleLLVMOptimizer=$(DotNetBuildMonoBundleLLVMOptimizer)</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(DotNetBuildAllRuntimePacks)' != ''">$(InnerBuildArgs) /p:DotNetBuildAllRuntimePacks=$(DotNetBuildAllRuntimePacks)</InnerBuildArgs>
+      <InnerBuildArgs Condition="'$(DotNetBuildPass)' != ''">$(InnerBuildArgs) /p:DotNetBuildPass=$(DotNetBuildPass)</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(PgoInstrument)' == 'true'">$(InnerBuildArgs) $(FlagParameterPrefix)pgoinstrument</InnerBuildArgs>
 
       <!-- This prop needs to be passed to the inner build manually as the BaseInnerSourceBuildCommand gets overridden above -->
@@ -83,6 +84,7 @@
       <InnerBuildArgs Condition="'$(RestoreConfigFile)' != ''">$(InnerBuildArgs) /p:RestoreConfigFile=$(RestoreConfigFile)</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(ForceDryRunSigning)' != ''">$(InnerBuildArgs) /p:ForceDryRunSigning=$(ForceDryRunSigning)</InnerBuildArgs>
       <InnerBuildArgs Condition="'$(DefaultArtifactVisibility)' != ''">$(InnerBuildArgs) /p:DefaultArtifactVisibility=$(DefaultArtifactVisibility)</InnerBuildArgs>
+      <InnerBuildArgs Condition="'$(DotNetEsrpToolPath)' != ''">$(InnerBuildArgs) /p:DotNetEsrpToolPath=$(DotNetEsrpToolPath)</InnerBuildArgs>
 
       <!-- Pass locations for assets, packages and symbols -->
       <InnerBuildArgs Condition="'$(SourceBuiltAssetsDir)' != ''">$(InnerBuildArgs) /p:SourceBuiltAssetsDir=$(SourceBuiltAssetsDir)</InnerBuildArgs>

eng/Subsets.props

@@ -67,6 +67,9 @@
     <!-- In source build, mono is only supported as primary runtime flavor. -->
     <DefaultSubsets Condition="'$(DotNetBuildSourceOnly)' == 'true' and '$(PrimaryRuntimeFlavor)' != 'Mono'">clr+libs+tools+host+packs</DefaultSubsets>
     <DefaultSubsets Condition="'$(DotNetBuildMonoCrossAOT)' == 'true'">mono+packs</DefaultSubsets>
+
+    <!-- In the Win-x86 BuildPass2 job in the VMR, we want to build the cross-OS DACs and pack them. -->
+    <DefaultSubsets Condition="'$(DotNetBuildPass)' == '2' and '$(TargetRid)' == 'win-x86'">crossdacpack</DefaultSubsets>
   </PropertyGroup>
 
   <PropertyGroup>
@@ -95,8 +98,8 @@
   </PropertyGroup>
 
   <PropertyGroup>
-    <DefaultCoreClrSubsets>clr.native+linuxdac+clr.corelib+clr.tools+clr.nativecorelib+clr.packages+clr.nativeaotlibs+clr.crossarchtools+host.native</DefaultCoreClrSubsets>
-    <DefaultCoreClrSubsets Condition="'$(PackageRID)' == 'linux-armel'">clr.native+linuxdac+clr.corelib+clr.tools+clr.nativecorelib+clr.packages+clr.nativeaotlibs+clr.crossarchtools</DefaultCoreClrSubsets>
+    <DefaultCoreClrSubsets>clr.native+clr.corelib+clr.tools+clr.nativecorelib+clr.packages+clr.nativeaotlibs+clr.crossarchtools+host.native</DefaultCoreClrSubsets>
+    <DefaultCoreClrSubsets Condition="'$(PackageRID)' == 'linux-armel'">clr.native+clr.corelib+clr.tools+clr.nativecorelib+clr.packages+clr.nativeaotlibs+clr.crossarchtools</DefaultCoreClrSubsets>
     <!-- Even on platforms that do not support the CoreCLR runtime, we still want to build ilasm/ildasm. -->
     <DefaultCoreClrSubsets Condition="'$(RuntimeFlavor)' != 'CoreCLR'">clr.iltools+clr.packages</DefaultCoreClrSubsets>
 
@@ -194,7 +197,7 @@
     <SubsetName Include="Clr.Tools" Description="Managed tools that support CoreCLR development and testing." />
     <SubsetName Include="Clr.ToolsTests" OnDemand="true" Description="Unit tests for the clr.tools subset." />
     <SubsetName Include="Clr.Packages" Description="The projects that produce NuGet packages for the CoreCLR runtime, crossgen, and IL tools." />
-    <SubsetName Include="LinuxDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" Description="The cross-OS Windows->libc-based Linux DAC. Skipped on x86." />
+    <SubsetName Include="LinuxDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" OnDemand="true" Description="The cross-OS Windows->libc-based Linux DAC. Skipped on x86." />
     <SubsetName Include="AlpineDac" Condition="$([MSBuild]::IsOsPlatform(Windows))" OnDemand="true" Description="The cross-OS Windows->musl-libc-based Linux DAC. Skipped on x86" />
     <SubsetName Include="CrossDacPack" OnDemand="true"
             Description="Packaging of cross OS DAC. Requires all assets needed to be present at a folder specified by $(CrossDacArtifactsDir). See 'Microsoft.CrossOsDiag.Private.CoreCLR.proj' for details." />

eng/extract-for-crossdac.ps1

@@ -0,0 +1,25 @@
+[CmdletBinding()]
+param (
+    [Parameter(Mandatory)]
+    [string]
+    $DownloadDirectory,
+    [Parameter(Mandatory)]
+    [string]
+    $ExtractDirectory
+)
+
+Add-Type -Assembly 'System.IO.Compression.FileSystem'
+
+Write-Host "Looking for packages under $DownloadDirectory"
+
+foreach ($file in Get-ChildItem $DownloadDirectory -Recurse -Filter '*.nupkg') {
+    Write-Host "Found Package: $($file.FullName)"
+    if ($file.Name -match '^(?<id>Microsoft.NETCore.App.Runtime.linux(-musl)?-((arm(64)?)|x64)).(?<ver>.+).nupkg$') {
+        $id = $matches['id']
+        $ver = $matches['ver']
+        Write-Host "Extracting Package: $id $ver to $ExtractDirectory/$($id.ToLowerInvariant())/$ver"
+        [System.IO.Compression.ZipFile]::ExtractToDirectory($file.FullName, "$ExtractDirectory/$($id.ToLowerInvariant())/$ver")
+    } else {
+        throw "Unexpected file name: $($file.Name)"
+    }
+}

eng/native/sign-with-dac-certificate.ps1

@@ -6,19 +6,25 @@ param(
     [Parameter(ValueFromRemainingArguments=$true)][string[]]$filesToSign
 )
 
-$inputFile = Get-Content -Raw $PSScriptRoot/signing/input.template.json | ConvertFrom-Json
-$inputFile.SignBatches.SignRequestFiles = $filesToSign | ForEach-Object {
+[array]$signEntries = $filesToSign | ForEach-Object {
     @{
         SourceLocation = $_
     }
 }
 
+$inputFile = Get-Content -Raw $PSScriptRoot/signing/input.template.json | ConvertFrom-Json
+$inputFile.SignBatches[0].SignRequestFiles = $signEntries
+
 $inputJson = [System.IO.Path]::GetTempFileName()
 # Our JSON goes up to 6 levels deep, so we need to set the depth to 6
 # to successfully round-trip our JSON through ConvertTo-Json
 $inputFile | ConvertTo-Json -Depth 6 | Out-File -FilePath $inputJson -Encoding utf8
 
-$outputJson = Resolve-Path "$PSScriptRoot/../../artifacts/log/Release/signing-$(New-Guid).json.log"
+$outputJson = "$PSScriptRoot/../../artifacts/log/Release/signing-$(New-Guid).json.log"
+
+New-Item $outputJson -ItemType File
+
+$outputJson = Resolve-Path $outputJson
 
 Write-Host "Signing files with DAC certificate"
 Write-Host "Logging output to $outputJson"

eng/pipelines/coreclr/templates/install-diagnostic-certs.yml

@@ -1,16 +1,9 @@
 parameters:
-  isOfficialBuild:
-    type: boolean
-    displayName: 'Is Official Build'
-  certNames:
-    type: array
-    displayName: 'Certificate Name'
-  vaultName:
-    type: string
-    displayName: 'Key Vault Name'
-  azureSubscription:
-    type: string
-    displayName: 'Azure Subscription'
+  isOfficialBuild: false
+  certNames: []
+  vaultName: ''
+  azureSubscription: ''
+  scriptRoot: '$(Build.SourcesDirectory)'
 
 steps:
 - ${{ if and(eq(parameters.isOfficialBuild, true), ne(variables['Build.Reason'], 'PullRequest'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/reltest/')), not(endsWith(variables['Build.SourceBranch'], '-staging'))) }}:
@@ -29,6 +22,7 @@ steps:
       $signArgs = '/p:DotNetEsrpToolPath=$(esrpclient.toolpath)\$(esrpclient.toolname)'
       echo "##vso[task.setvariable variable=_SignDiagnosticFilesArgs;]$signArgs"
     displayName: 'Install diagnostic certificates'
+    workingDirectory: ${{ parameters.scriptRoot }}
     env:
       ${{ each cert in parameters.certNames }}:
         ${{ cert }}: $(${{ cert }})

eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml

@@ -1,11 +1,11 @@
 parameters:
-  isOfficialBuild:
-    type: boolean
-    displayName: 'Is Official Build'
+  isOfficialBuild: false
+  scriptRoot: '$(Build.SourcesDirectory)'
 
 steps:
 - ${{ if and(eq(parameters.isOfficialBuild, true), ne(variables['Build.Reason'], 'PullRequest'), or(startswith(variables['Build.SourceBranch'], 'refs/heads/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/internal/release/'), startswith(variables['Build.SourceBranch'], 'refs/heads/reltest/')), not(endsWith(variables['Build.SourceBranch'], '-staging'))) }}:
   - powershell: |
       eng/pipelines/remove-diagnostic-certs.ps1 "$(DacCertificateThumbprints)"
+    workingDirectory: ${{ parameters.scriptRoot }}
     displayName: 'Remove Diagnostic Certificates'
     condition: always()

eng/pipelines/runtime-official.yml

@@ -151,7 +151,6 @@ extends:
 
       #
       # Build all runtime packs for Linux and Linux musl
-      # Upload CoreCLR runtime for CrossDac packaging
       #
       - template: /eng/pipelines/common/platform-matrix.yml
         parameters:
@@ -171,26 +170,10 @@ extends:
             isOfficialBuild: ${{ variables.isOfficialBuild }}
             timeoutInMinutes: 120
             postBuildSteps:
-              # Upload libcoreclr.so for CrossDac packaging
-              - task: CopyFiles@2
-                displayName: Gather runtime for CrossDac
-                inputs:
-                  SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr/$(osGroup).$(archType).$(_BuildConfig)
-                  Contents: libcoreclr.so
-                  TargetFolder: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts/$(osGroup)$(osSubgroup).$(archType).$(_BuildConfig)/$(crossDacHostArch)
-              - task: 1ES.PublishBuildArtifacts@1
-                displayName: Publish runtime for CrossDac
-                inputs:
-                  PathtoPublish: $(Build.SourcesDirectory)/artifacts/CoreCLRCrossDacArtifacts
-                  PublishLocation: Container
-                  ArtifactName: CoreCLRCrossDacArtifacts
-
               # Upload the results.
               - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
                 parameters:
                   name: $(osGroup)$(osSubgroup)_$(archType)
-            extraVariablesTemplates:
-              - template: /eng/pipelines/coreclr/templates/crossdac-hostarch.yml
 
       #
       # Build and Pack CrossDac
@@ -203,17 +186,21 @@ extends:
           - windows_x64
           jobParameters:
             templatePath: 'templates-official'
-            buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossDacArtifactsDir=$(crossDacArtifactsPath)
+            buildArgs: -s crossdacpack -c $(_BuildConfig) /p:CrossRuntimeExtractionRoot=$(CrossRuntimeExtractionRoot) $(_SignDiagnosticFilesArgs)
             nameSuffix: CrossDac
             isOfficialBuild: ${{ variables.isOfficialBuild }}
             timeoutInMinutes: 120
             preBuildSteps:
-            - task: DownloadBuildArtifacts@0
-              displayName: Download Runtimes for CrossDac packaging
+            - task: DownloadPipelineArtifact@2
+              displayName: Download runtime packs for CrossDac
               inputs:
-                artifactName: $(crossDacArtifactsContainer)
-                downloadPath: $(crossDacArtifactsBasePath)
-                checkDownloadedFiles: true
+                artifact: 'IntermediateArtifacts'
+                path: $(Build.SourcesDirectory)/artifacts/RuntimeDownload
+                patterns: |
+                  IntermediateArtifacts/linux_*/Shipping/Microsoft.NETCore.App.Runtime.linux-*.nupkg
+                  !IntermediateArtifacts/linux_*/Shipping/Microsoft.NETCore.App.Runtime.linux-*.symbols.nupkg
+            - powershell: $(Build.SourcesDirectory)/eng/extract-for-crossdac.ps1 -DownloadDirectory $(Build.SourcesDirectory)/artifacts/RuntimeDownload -ExtractDirectory $(CrossRuntimeExtractionRoot)
+              displayName: Extract runtime packs
             - template: /eng/pipelines/coreclr/templates/install-diagnostic-certs.yml
               parameters:
                 isOfficialBuild: ${{ variables.isOfficialBuild }}
@@ -222,22 +209,10 @@ extends:
                 - 'dotnet-diagnostics-esrp-pki-onecert'
                 vaultName: 'clrdiag-esrp-id'
                 azureSubscription: 'diagnostics-esrp-kvcertuser'
-            - template: /eng/pipelines/common/templates/global-build-step.yml
-              parameters:
-                buildArgs: -s linuxdac+alpinedac -c $(_BuildConfig) $(_SignDiagnosticFilesArgs)
-                archParameter: -arch x64,x86,arm,arm64
+            postBuildSteps:
             - template: /eng/pipelines/coreclr/templates/remove-diagnostic-certs.yml
               parameters:
                 isOfficialBuild: ${{ variables.isOfficialBuild }}
-            - task: CopyFiles@2
-              displayName: Gather CrossDacs
-              inputs:
-                SourceFolder: $(Build.SourcesDirectory)/artifacts/bin/coreclr
-                Contents: |
-                  **
-                  !**\sharedFramework\**
-                TargetFolder: $(crossDacArtifactsPath)
-            postBuildSteps:
             # Save packages using the prepare-signed-artifacts format.
             # CrossDac packages are expected to be in the windows_x64 folder.
             - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
@@ -251,12 +226,10 @@ extends:
             - build_linux_musl_arm_release_AllRuntimes
             - build_linux_musl_arm64_release_AllRuntimes
           variables:
-          - name: crossDacArtifactsContainer
-            value: CoreCLRCrossDacArtifacts
-          - name: crossDacArtifactsBasePath
-            value: $(Build.StagingDirectory)/CrossDac
-          - name: crossDacArtifactsPath
-            value: $(crossDacArtifactsBasePath)/$(crossDacArtifactsContainer)
+          - name: CrossRuntimeExtractionRoot
+            value: $(Build.SourcesDirectory)/artifacts/CrossDac
+          - name: _SignDiagnosticFilesArgs
+            value: ''
 
       #
       # Build All runtime packs for mobile platforms

src/coreclr/.nuget/Microsoft.CrossOsDiag.Private.CoreCLR/Microsoft.CrossOsDiag.Private.CoreCLR.pkgproj

@@ -4,18 +4,19 @@
     <IsShipping>false</IsShipping>
     <CreatePackedPackage>false</CreatePackedPackage>
     <PackageDescription>Private transport package for .NET Core cross OS diagnostic tooling.</PackageDescription>
+    <CrossDacArtifactsDir>$(ArtifactsBinDir)/coreclr/$(CrossDacOutputDir)</CrossDacArtifactsDir>
   </PropertyGroup>
 
   <ItemGroup>
     <!-- Intentional rather than globbing here, even if very manual, to make sure we publish the Cross OS Dac -->
-    <NativeBinary Include="$(CrossDacBinRoot)/mscordaccore.dll" />
-    <NativeBinary Include="$(CrossDacBinRoot)/mscordbi.dll" />
+    <NativeBinary Include="$(CrossDacArtifactsDir)/mscordaccore.dll" />
+    <NativeBinary Include="$(CrossDacArtifactsDir)/mscordbi.dll" />
     <!-- The infrastructure, and this package is only for indexing. Avoid creating a transport package and just let this one have the symbols itself -->
     <File Include="@(NativeBinary -> '%(RootDir)%(Directory)/%(Filename).pdb')">
       <TargetPath>runtimes/$(PackageTargetRuntime)/native</TargetPath>
     </File>
 
-    <NativeBinary Include="$(CrossDacBinRoot)/libcoreclr.so" />
+    <NativeBinary Include="$(CoreCLRRuntimePath)/libcoreclr.so" />
   </ItemGroup>
 
   <Import Project="$([MSBuild]::GetPathOfFileAbove(Directory.Build.targets))" />