Superpmi on Microbenchmarks (dotnet#47900)

* Superpmi on Microbenchmarks * Fix the mch file path name * distribute the benchmarks in 30 partitions * Set input directory for benchmarks * Some fixes to superpmi_benchmarks Also update the GUID so we don't overwrite the existing collections. * fix the name of partition_index and partition_count * Point the core_root path to the superpmi script * fix python to invoke for setup * Add verbosity and include all benchmarks * Fix the benchmarks invocation * use benchmarks_ci.py script * run all benchmarks * fix the performance source code path * see why dotnet install fails * Comment all jobs except benchmarks * update the right fork * Switch back to doing installing dotnet on azure machine * Put dotnet in script * fix dumpMap, revert change in superpmi.py * Produce artifacts in temp folder - Disable mcs -strip for now - Pass the JitName variable * Experimental: Exit on failure * Revert "Produce artifacts in temp folder" This reverts commit afdfbd4b03a684d780ef06f644dba0dcf0621438. * Use JitName * Use workitem folder instead of correlation * fix typo in WorkItemDirectory * Set the payload directory * print error message before exiting * fix some linux issues * Make dotnet executable * resolve merge conflicts * fix typo from merge conflict * add logging around chmod * fix the is_windows condition * cleanup and disable linux arm/arm64 * remove the unwanted parameter
Nisden · Feb 13, 2021 · d4a9b6b · d4a9b6b
1 parent 80b3a3f
commit d4a9b6b
Show file tree

Hide file tree

Showing 7 changed files with 506 additions and 101 deletions.
diff --git a/eng/pipelines/coreclr/superpmi.yml b/eng/pipelines/coreclr/superpmi.yml
@@ -94,6 +94,28 @@ jobs:
       collectionType: crossgen
       collectionName: libraries
 
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml
+    buildConfig: checked
+    platforms:
+    # Linux tests are built on the OSX machines.
+    # - OSX_x64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_x64
+    - windows_x64
+    - windows_x86
+    - windows_arm64
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
+    helixQueueGroup: ci
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      testGroup: outerloop
+      liveLibrariesBuildConfig: Release
+      collectionType: pmi
+      collectionName: tests
+
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml
@@ -123,8 +145,9 @@ jobs:
     platforms:
     # Linux tests are built on the OSX machines.
     # - OSX_x64
-    - Linux_arm
-    - Linux_arm64
+    #TODO: Need special handling of running "benchmark build" from inside TMP folder on helix machine.
+    # - Linux_arm
+    # - Linux_arm64
     - Linux_x64
     - windows_x64
     - windows_x86
@@ -135,6 +158,5 @@ jobs:
     jobParameters:
       testGroup: outerloop
       liveLibrariesBuildConfig: Release
-      collectionType: pmi
-      collectionName: tests
-
+      collectionType: run
+      collectionName: benchmarks
diff --git a/eng/pipelines/coreclr/templates/run-superpmi-job.yml b/eng/pipelines/coreclr/templates/run-superpmi-job.yml
@@ -87,6 +87,9 @@ jobs:
     - ${{ if eq(parameters.collectionName, 'libraries') }}:
       - name: InputDirectory
         value: '$(Core_Root_Dir)'
+    - ${{ if eq(parameters.collectionName, 'benchmarks') }}:
+      - name: InputDirectory
+        value: '$(Core_Root_Dir)'
     - ${{ if eq(parameters.collectionName, 'tests') }}:
       - name: InputDirectory
         value: '$(managedTestArtifactRootFolderPath)'
@@ -103,7 +106,7 @@ jobs:
     steps:
     - ${{ parameters.steps }}
 
-    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi-setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 50 # size in MB
+    - script: $(PythonScript) $(Build.SourcesDirectory)/src/coreclr/scripts/superpmi_setup.py -source_directory $(Build.SourcesDirectory) -core_root_directory $(Core_Root_Dir) -arch $(archType) -mch_file_tag $(MchFileTag) -input_directory $(InputDirectory) -collection_name $(CollectionName) -collection_type $(CollectionType) -max_size 50 # size in MB
       displayName: ${{ format('SuperPMI setup ({0})', parameters.osGroup) }}
 
       # Run superpmi collection in helix

diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h
@@ -32,11 +32,11 @@
 //////////////////////////////////////////////////////////////////////////////////////////////////////////
 //
 
-constexpr GUID JITEEVersionIdentifier = { /* 6ca59d19-13a4-44f7-a184-e9cd1e8f57f8 */
-    0x6ca59d19,
-    0x13a4,
-    0x44f7,
-    { 0xa1, 0x84, 0xe9, 0xcd, 0x1e, 0x8f, 0x57, 0xf8 }
+constexpr GUID JITEEVersionIdentifier = { /* af37688b-d4e5-4a41-a7ee-701728a470aa */
+    0xaf37688b,
+    0xd4e5,
+    0x4a41,
+    {0xa7, 0xee, 0x70, 0x17, 0x28, 0xa4, 0x70, 0xaa}
 };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

diff --git a/src/coreclr/scripts/superpmi.proj b/src/coreclr/scripts/superpmi.proj
@@ -1,5 +1,31 @@
 <Project Sdk="Microsoft.DotNet.Helix.Sdk" DefaultTargets="Test">
 
+   <!--
+    This is useful for local testing to print the produced helix items
+    To use this when you are changing how items are produced, uncomment the target
+    and replace the Project item at the top of the file with this:
+    <Project DefaultTargets="printItems">
+
+    Once you've done that you can run this to see the results:
+    dotnet msbuild .\superpmi.proj /v:n
+   -->
+
+   <!-- <PropertyGroup>
+     <HelixTargetQueues>Some_Queue</HelixTargetQueues>
+     <InputArtifacts>D:\git\runtime\workitem\pmiAssembliesDirectory\libraries</InputArtifacts>
+     <MchFileTag>x64.checked</MchFileTag>
+     <CollectionName>benchmarks</CollectionName>
+     <CollectionType>pmi</CollectionType>
+   </PropertyGroup>
+  <Target Name="printItems">
+        <Message Text="@(HelixWorkItem -> 'name: %(HelixWorkItem.Identity)
+     dir: %(HelixWorkItem.PayloadDirectory)
+     pre: %(HelixWorkItem.PreCommands)
+     command: %(HelixWorkItem.Command)
+     post: %(HelixWorkItem.PostCommands)
+     timeout: %(HelixWorkItem.Timeout)  '"/>
+  </Target> -->
+
   <PropertyGroup Condition="'$(AGENT_OS)' == 'Windows_NT'">
     <FileSeparatorChar>\</FileSeparatorChar>
   </PropertyGroup>
@@ -21,6 +47,10 @@
     <PmiAssembliesPayload>$(WorkItemDirectory)\pmiAssembliesDirectory</PmiAssembliesPayload>
     <PmiAssembliesDirectory>%HELIX_WORKITEM_PAYLOAD%\binaries</PmiAssembliesDirectory>
     <SuperPMIDirectory>%HELIX_CORRELATION_PAYLOAD%\superpmi</SuperPMIDirectory>
+
+    <!-- Related to Benchmarks -->
+    <PerformanceDirectory>%HELIX_WORKITEM_PAYLOAD%\performance</PerformanceDirectory>
+
     <OutputMchPath>%HELIX_WORKITEM_UPLOAD_ROOT%</OutputMchPath>
     <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
     <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)\artifacts\helixresults</HelixResultsDestinationDir>
@@ -31,6 +61,10 @@
     <PmiAssembliesPayload>$(WorkItemDirectory)/pmiAssembliesDirectory</PmiAssembliesPayload>
     <PmiAssembliesDirectory>$HELIX_WORKITEM_PAYLOAD/binaries</PmiAssembliesDirectory>
     <SuperPMIDirectory>$HELIX_CORRELATION_PAYLOAD/superpmi</SuperPMIDirectory>
+
+    <!-- Related to Benchmarks -->
+    <PerformanceDirectory>$HELIX_WORKITEM_PAYLOAD/performance</PerformanceDirectory>
+
     <OutputMchPath>$HELIX_WORKITEM_UPLOAD_ROOT</OutputMchPath>
     <!-- Workaround until https://github.com/dotnet/arcade/pull/6179 is not available -->
     <HelixResultsDestinationDir>$(BUILD_SOURCESDIRECTORY)/artifacts/helixresults</HelixResultsDestinationDir>
@@ -41,6 +75,10 @@
     <WorkItemCommand>$(Python) $(WorkItemCommand) -assemblies $(PmiAssembliesDirectory) -arch $(Architecture) -build_type $(BuildConfig) -core_root $(SuperPMIDirectory)</WorkItemCommand>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(CollectionName)' == 'benchmarks'">
+    <WorkItemCommand>$(Python) $(SuperPMIDirectory)/superpmi_benchmarks.py -performance_directory $(PerformanceDirectory) -superpmi_directory $(SuperPMIDirectory) -core_root $(SuperPMIDirectory) -arch $(Architecture)</WorkItemCommand>
+  </PropertyGroup>
+
   <PropertyGroup>
     <EnableAzurePipelinesReporter>false</EnableAzurePipelinesReporter>
     <EnableXUnitReporter>false</EnableXUnitReporter>
@@ -68,12 +106,48 @@
     </HelixCorrelationPayload>
   </ItemGroup>
 
-  <ItemGroup>
+  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks'">
     <PartitionDirectories Include="$([System.IO.Directory]::GetDirectories($(InputArtifacts)))"/>
     <Partition Include="@(PartitionDirectories -> '%(Filename)')" PmiAssemblies="%(Filename)" PartitionId="%(Filename)" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <PartitionCount>30</PartitionCount>
+  </PropertyGroup>
   <ItemGroup>
+    <BDN_Partition Include="Partition0" Index="0" />
+    <BDN_Partition Include="Partition1" Index="1" />
+    <BDN_Partition Include="Partition2" Index="2" />
+    <BDN_Partition Include="Partition3" Index="3" />
+    <BDN_Partition Include="Partition4" Index="4" />
+    <BDN_Partition Include="Partition5" Index="5" />
+    <BDN_Partition Include="Partition6" Index="6" />
+    <BDN_Partition Include="Partition7" Index="7" />
+    <BDN_Partition Include="Partition8" Index="8" />
+    <BDN_Partition Include="Partition9" Index="9" />
+    <BDN_Partition Include="Partition10" Index="10" />
+    <BDN_Partition Include="Partition11" Index="11" />
+    <BDN_Partition Include="Partition12" Index="12" />
+    <BDN_Partition Include="Partition13" Index="13" />
+    <BDN_Partition Include="Partition14" Index="14" />
+    <BDN_Partition Include="Partition15" Index="15" />
+    <BDN_Partition Include="Partition16" Index="16" />
+    <BDN_Partition Include="Partition17" Index="17" />
+    <BDN_Partition Include="Partition18" Index="18" />
+    <BDN_Partition Include="Partition19" Index="19" />
+    <BDN_Partition Include="Partition20" Index="20" />
+    <BDN_Partition Include="Partition21" Index="21" />
+    <BDN_Partition Include="Partition22" Index="22" />
+    <BDN_Partition Include="Partition23" Index="23" />
+    <BDN_Partition Include="Partition24" Index="24" />
+    <BDN_Partition Include="Partition25" Index="25" />
+    <BDN_Partition Include="Partition26" Index="26" />
+    <BDN_Partition Include="Partition27" Index="27" />
+    <BDN_Partition Include="Partition28" Index="28" />
+    <BDN_Partition Include="Partition29" Index="29" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(CollectionName)' != 'benchmarks'">
     <HelixWorkItem Include="@(Partition)">
       <OutputFileName>$(CollectionName).$(CollectionType).%(HelixWorkItem.PartitionId).$(MchFileTag)</OutputFileName>
       <PayloadDirectory>$(PmiAssembliesPayload)$(FileSeparatorChar)$(CollectionName)$(FileSeparatorChar)%(HelixWorkItem.PmiAssemblies)</PayloadDirectory>
@@ -82,30 +156,14 @@
       <DownloadFilesFromResults>%(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log</DownloadFilesFromResults>
     </HelixWorkItem>
   </ItemGroup>
-    <!--
-    This is useful for local testing to print the produced helix items
-    To use this when you are changing how items are produced, uncomment the target
-    and replace the Project item at the top of the file with this:
-    <Project DefaultTargets="printItems">
-
-    Once you've done that you can run this to see the results:
-    dotnet msbuild .\superpmi.proj /v:n
-   -->
-<!--    
-   <PropertyGroup>
-     <HelixTargetQueues>Some_Queue</HelixTargetQueues>
-     <InputArtifacts>D:\git\runtime\workitem\pmiAssembliesDirectory\libraries</InputArtifacts>
-     <MchFileTag>x64.checked</MchFileTag>
-     <CollectionName>libraries</CollectionName>
-   </PropertyGroup>
-  <Target Name="printItems">
-        <Message Text="@(HelixWorkItem -> 'name: %(HelixWorkItem.Identity)
-     dir: %(HelixWorkItem.PayloadDirectory)
-     pre: %(HelixWorkItem.PreCommands)
-     command: %(HelixWorkItem.Command)
-     post: %(HelixWorkItem.PostCommands)
-     timeout: %(HelixWorkItem.Timeout)  '"/>
-  </Target> -->
-
 
+  <ItemGroup Condition="'$(CollectionName)' == 'benchmarks'">
+    <HelixWorkItem Include="@(BDN_Partition)">
+      <OutputFileName>$(CollectionName).$(CollectionType).%(HelixWorkItem.Index).$(MchFileTag)</OutputFileName>
+      <PayloadDirectory>$(WorkItemDirectory)</PayloadDirectory>
+      <Command>$(WorkItemCommand) -partition_count $(PartitionCount) -partition_index %(HelixWorkItem.Index) -output_mch_path $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).mch -log_file $(OutputMchPath)$(FileSeparatorChar)%(OutputFileName).log</Command>
+      <Timeout>$(WorkItemTimeout)</Timeout>
+      <DownloadFilesFromResults>%(OutputFileName).mch;%(OutputFileName).mch.mct;%(OutputFileName).log</DownloadFilesFromResults>
+    </HelixWorkItem>
+  </ItemGroup>
 </Project>
diff --git a/src/coreclr/scripts/superpmi.py b/src/coreclr/scripts/superpmi.py
@@ -543,12 +543,13 @@ class TempDir:
 
         Use with: "with TempDir() as temp_dir" to change to that directory and then automatically
         change back to the original working directory afterwards and remove the temporary
-        directory and its contents (if args.skip_cleanup is False).
+        directory and its contents (if skip_cleanup is False).
     """
 
-    def __init__(self, path=None):
+    def __init__(self, path=None, skip_cleanup=False):
         self.mydir = tempfile.mkdtemp() if path is None else path
         self.cwd = None
+        self._skip_cleanup = skip_cleanup
 
     def __enter__(self):
         self.cwd = os.getcwd()
@@ -557,10 +558,7 @@ def __enter__(self):
 
     def __exit__(self, exc_type, exc_val, exc_tb):
         os.chdir(self.cwd)
-        # Note: we are using the global `args`, not coreclr_args. This works because
-        # the `skip_cleanup` argument is not processed by CoreclrArguments, but is
-        # just copied there.
-        if not args.skip_cleanup:
+        if not self._skip_cleanup:
             shutil.rmtree(self.mydir)
 
 
@@ -758,7 +756,7 @@ def collect(self):
         passed = False
 
         try:
-            with TempDir(self.coreclr_args.temp_dir) as temp_location:
+            with TempDir(self.coreclr_args.temp_dir, self.coreclr_args.skip_cleanup) as temp_location:
                 # Setup all of the temp locations
                 self.base_fail_mcl_file = os.path.join(temp_location, "basefail.mcl")
                 self.base_mch_file = os.path.join(temp_location, "base.mch")
@@ -1573,7 +1571,7 @@ def replay_with_asm_diffs(self):
         files_with_asm_diffs = []
         files_with_replay_failures = []
 
-        with TempDir(self.coreclr_args.temp_dir) as temp_location:
+        with TempDir(self.coreclr_args.temp_dir, self.coreclr_args.skip_cleanup) as temp_location:
             logging.debug("")
             logging.debug("Temp Location: %s", temp_location)
             logging.debug("")