Fix SuperPMI collections (dotnet#107588)

* Fix SuperPMI collections 1. Remove FSharp benchmark dll from realworld collection, since it no longer has any benchmarks. 2. Properly pass `-cross` switch when building CORE_ROOT. Propagate `crossBuild` more widely through the scripts. * Pass `container` to xplat-job in run-superpmi-collect-job * Better error message * More `container` changes
huoyaoyuan · Sep 10, 2024 · e4f93db · e4f93db
1 parent 4a055bd
commit e4f93db
Show file tree

Hide file tree

Showing 10 changed files with 40 additions and 16 deletions.
diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh
@@ -84,7 +84,7 @@ initDistroRidGlobal()
     if [ -n "${rootfsDir}" ]; then
         # We may have a cross build. Check for the existence of the rootfsDir
         if [ ! -e "${rootfsDir}" ]; then
-            echo "Error: rootfsDir has been passed, but the location is not valid."
+            echo "Error: rootfsDir has been passed (${rootfsDir}), but the location is not valid."
             exit 1
         fi
     fi

diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -35,6 +35,7 @@ parameters:
   preBuildSteps: []
   templatePath: 'templates'
   templateContext: ''
+  disableComponentGovernance: ''
 
 jobs:
 - template: /eng/common/${{ parameters.templatePath }}/job/job.yml

diff --git a/eng/pipelines/coreclr/superpmi-collect-test.yml b/eng/pipelines/coreclr/superpmi-collect-test.yml
@@ -65,6 +65,7 @@ extends:
                 parameters:
                   buildArgs: -s clr.spmi -c $(_BuildConfig)
                   archParameter: -arch x64
+                  displayName: Build SuperPMI
               - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:
@@ -106,6 +107,7 @@ extends:
                   buildArgs: -s clr.spmi -c $(_BuildConfig)
                   archParameter: -arch x64
                   container: linux_x64
+                  displayName: Build SuperPMI
               - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:

diff --git a/eng/pipelines/coreclr/superpmi-collect.yml b/eng/pipelines/coreclr/superpmi-collect.yml
@@ -86,6 +86,7 @@ extends:
                 parameters:
                   buildArgs: -s clr.spmi -c $(_BuildConfig)
                   archParameter: -arch x64
+                  displayName: Build SuperPMI
               - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:
@@ -127,6 +128,7 @@ extends:
                   buildArgs: -s clr.spmi -c $(_BuildConfig)
                   archParameter: -arch x64
                   container: linux_x64
+                  displayName: Build SuperPMI
               - template: /eng/pipelines/coreclr/templates/build-native-test-assets-step.yml
               - template: /eng/pipelines/common/upload-artifact-step.yml
                 parameters:

diff --git a/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-asmdiffs-checked-release-job.yml
@@ -9,6 +9,7 @@ parameters:
   archType: ''                    # required -- targeting CPU architecture
   osGroup: ''                     # required -- operating system for the job
   osSubgroup: ''                  # optional -- operating system subgroup
+  crossBuild: ''                  # optional -- 'true' if this is a cross-build
   continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
   dependsOn: ''                   # optional -- dependencies of the job
   timeoutInMinutes: 320           # optional -- timeout for the job
@@ -24,6 +25,7 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    crossBuild: ${{ parameters.crossBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true

diff --git a/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml
@@ -9,6 +9,7 @@ parameters:
   archType: ''                    # required -- targeting CPU architecture
   osGroup: ''                     # required -- operating system for the job
   osSubgroup: ''                  # optional -- operating system subgroup
+  crossBuild: ''                  # optional -- 'true' if this is a cross-build
   continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
   dependsOn: ''                   # optional -- dependencies of the job
   timeoutInMinutes: 320           # optional -- timeout for the job
@@ -25,6 +26,8 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    container: ${{ parameters.container }}
+    crossBuild: ${{ parameters.crossBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true
@@ -114,7 +117,6 @@ jobs:
       clean: all
     pool:
       ${{ parameters.pool }}
-    container: ${{ parameters.container }}
     steps:
     - ${{ parameters.steps }}
 

diff --git a/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-diffs-job.yml
@@ -9,6 +9,7 @@ parameters:
   archType: ''                    # required -- targeting CPU architecture
   osGroup: ''                     # required -- operating system for the job
   osSubgroup: ''                  # optional -- operating system subgroup
+  crossBuild: ''                  # optional -- 'true' if this is a cross-build
   continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
   dependsOn: ''                   # optional -- dependencies of the job
   timeoutInMinutes: 180           # optional -- timeout for the job
@@ -27,6 +28,7 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    crossBuild: ${{ parameters.crossBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true

diff --git a/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-replay-job.yml
@@ -9,6 +9,7 @@ parameters:
   archType: ''                    # required -- targeting CPU architecture
   osGroup: ''                     # required -- operating system for the job
   osSubgroup: ''                  # optional -- operating system subgroup
+  crossBuild: ''                  # optional -- 'true' if this is a cross-build
   continueOnError: 'false'        # optional -- determines whether to continue the build if the step errors
   dependsOn: ''                   # optional -- dependencies of the job
   timeoutInMinutes: 320           # optional -- timeout for the job
@@ -24,6 +25,7 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    crossBuild: ${{ parameters.crossBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     enableTelemetry: ${{ parameters.enableTelemetry }}
     enablePublishBuildArtifacts: true

diff --git a/eng/pipelines/coreclr/templates/superpmi-collect-job.yml b/eng/pipelines/coreclr/templates/superpmi-collect-job.yml
@@ -3,7 +3,9 @@ parameters:
   archType: ''
   osGroup: ''
   osSubgroup: ''
+  container: ''
   liveLibrariesBuildConfig: ''
+  crossBuild: ''
   variables: {}
   pool: ''
   runJobTemplate: '/eng/pipelines/coreclr/templates/run-superpmi-collect-job.yml'
@@ -24,6 +26,8 @@ jobs:
     archType: ${{ parameters.archType }}
     osGroup: ${{ parameters.osGroup }}
     osSubgroup: ${{ parameters.osSubgroup }}
+    container: ${{ parameters.container }}
+    crossBuild: ${{ parameters.crossBuild }}
     liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }}
     collectionType: ${{ parameters.collectionType }}
     collectionName: ${{ parameters.collectionName }}
@@ -33,7 +37,16 @@ jobs:
      - ${{ if eq(parameters.collectionName, 'coreclr_tests') }}:
         - 'coreclr_common_test_build_p1_AnyOS_AnyCPU_${{parameters.buildConfig }}'
 
-    variables: ${{ parameters.variables }}
+    variables:
+
+    - name: crossArg
+      value: ''
+    - ${{ if eq(parameters.crossBuild, true) }}:
+      - name: crossArg
+        value: '-cross'
+
+    - ${{ each variable in parameters.variables }}:
+      - ${{insert}}: ${{ variable }}
 
     steps:
     # Extra steps that will be passed to the superpmi template and run before sending the job to helix (all of which is done in the template)
@@ -72,6 +85,6 @@ jobs:
           displayName: 'generic managed test artifacts'
 
     # Create Core_Root
-    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg) /p:UsePublishedCrossgen2=false
+    - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly $(librariesOverrideArg) /p:UsePublishedCrossgen2=false
       displayName: Create Core_Root
       condition: succeeded()
diff --git a/src/coreclr/scripts/superpmi-collect.proj b/src/coreclr/scripts/superpmi-collect.proj
@@ -246,21 +246,19 @@
   </ItemGroup>
   <ItemGroup Condition="'$(CollectionName)' == 'realworld' and '$(AGENT_OS)' == 'Windows_NT'">
     <BDN_Partition Include="Partition0" BenchmarkPath="src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj" BenchmarkBinary="DemoBenchmarks.dll" />
-    <BDN_Partition Include="Partition1" BenchmarkPath="src\benchmarks\real-world\FSharp\FSharp.fsproj" BenchmarkBinary="FSharp.dll" />
-    <BDN_Partition Include="Partition2" BenchmarkPath="src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj" BenchmarkBinary="ILLinkBenchmarks.dll" />
-    <BDN_Partition Include="Partition3" BenchmarkPath="src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj" BenchmarkBinary="ImageSharp.Benchmarks.dll" />
-    <BDN_Partition Include="Partition4" BenchmarkPath="src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" BenchmarkBinary="Microsoft.ML.Benchmarks.dll" />
-    <BDN_Partition Include="Partition5" BenchmarkPath="src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj" BenchmarkBinary="CompilerBenchmarks.dll" />
-    <BDN_Partition Include="Partition6" BenchmarkPath="src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj" BenchmarkBinary="PowerShell.Benchmarks.dll" />
+    <BDN_Partition Include="Partition1" BenchmarkPath="src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj" BenchmarkBinary="ILLinkBenchmarks.dll" />
+    <BDN_Partition Include="Partition2" BenchmarkPath="src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj" BenchmarkBinary="ImageSharp.Benchmarks.dll" />
+    <BDN_Partition Include="Partition3" BenchmarkPath="src\benchmarks\real-world\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" BenchmarkBinary="Microsoft.ML.Benchmarks.dll" />
+    <BDN_Partition Include="Partition4" BenchmarkPath="src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj" BenchmarkBinary="CompilerBenchmarks.dll" />
+    <BDN_Partition Include="Partition5" BenchmarkPath="src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj" BenchmarkBinary="PowerShell.Benchmarks.dll" />
   </ItemGroup>  
   <ItemGroup Condition="'$(CollectionName)' == 'realworld' and '$(AGENT_OS)' != 'Windows_NT'">
     <BDN_Partition Include="Partition0" BenchmarkPath="src/benchmarks/real-world/bepuphysics2/DemoBenchmarks.csproj" BenchmarkBinary="DemoBenchmarks.dll" />
-    <BDN_Partition Include="Partition1" BenchmarkPath="src/benchmarks/real-world/FSharp/FSharp.fsproj" BenchmarkBinary="FSharp.dll" />
-    <BDN_Partition Include="Partition2" BenchmarkPath="src/benchmarks/real-world/ILLink/ILLinkBenchmarks.csproj" BenchmarkBinary="ILLinkBenchmarks.dll" />
-    <BDN_Partition Include="Partition3" BenchmarkPath="src/benchmarks/real-world/ImageSharp/ImageSharp.Benchmarks.csproj" BenchmarkBinary="ImageSharp.Benchmarks.dll" />
-    <BDN_Partition Include="Partition4" BenchmarkPath="src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj" BenchmarkBinary="Microsoft.ML.Benchmarks.dll" />
-    <BDN_Partition Include="Partition5" BenchmarkPath="src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj" BenchmarkBinary="CompilerBenchmarks.dll" />
-    <BDN_Partition Include="Partition6" BenchmarkPath="src/benchmarks/real-world/PowerShell.Benchmarks/PowerShell.Benchmarks.csproj" BenchmarkBinary="PowerShell.Benchmarks.dll" />
+    <BDN_Partition Include="Partition1" BenchmarkPath="src/benchmarks/real-world/ILLink/ILLinkBenchmarks.csproj" BenchmarkBinary="ILLinkBenchmarks.dll" />
+    <BDN_Partition Include="Partition2" BenchmarkPath="src/benchmarks/real-world/ImageSharp/ImageSharp.Benchmarks.csproj" BenchmarkBinary="ImageSharp.Benchmarks.dll" />
+    <BDN_Partition Include="Partition3" BenchmarkPath="src/benchmarks/real-world/Microsoft.ML.Benchmarks/Microsoft.ML.Benchmarks.csproj" BenchmarkBinary="Microsoft.ML.Benchmarks.dll" />
+    <BDN_Partition Include="Partition4" BenchmarkPath="src/benchmarks/real-world/Roslyn/CompilerBenchmarks.csproj" BenchmarkBinary="CompilerBenchmarks.dll" />
+    <BDN_Partition Include="Partition5" BenchmarkPath="src/benchmarks/real-world/PowerShell.Benchmarks/PowerShell.Benchmarks.csproj" BenchmarkBinary="PowerShell.Benchmarks.dll" />
   </ItemGroup>  
 
   <ItemGroup Condition="'$(CollectionName)' != 'benchmarks' and '$(CollectionName)' != 'realworld'">