Enable superpmi collection for libraries tests (dotnet#51507)

* Run SPMI collection on libraries-tests

* Only run for libraries-tests and change GUID

* fix error

* fix the collection name

* fix the timeoutInMinutes

* Download libraries test artifacts

* try to fix the libs+tests artifacts name

* Use artifacts name as is

* Fix the libraries test artifacts name

* Use librares_test.zip

* Modify the filter to unzip all *Tests.dll files

* Revert "Modify the filter to unzip all *Tests.dll files"

This reverts commit b62f713.

* Try to unzip libraries_zipped

* overwriteExistingFiles=true

* Fix the asset extension

- Also exclude files present in core_root

* fix artifacts name

* remove extra .

* Ignore permission error

* Copy common test files in CORE_ROOT

- And do not copy not *.Tests.dll" for PMI

* Perform collection for all libraries_tests assets

* Add UnicodeEncodeError EH

* Add PermissionError for copy2

* Copy all the test assests to core_root

* Make input_directory readonly

* make ch_mod recursive

* Add option of error_limit to superpmi.py

* Pass -failureLimit flag to parallel mode

* Revert "Only run for libraries-tests and change GUID"

This reverts commit 7f4cdda.

eng/pipelines/coreclr/superpmi.yml

@@ -41,6 +41,10 @@ jobs:
     - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
     jobParameters:
       testGroup: outerloop
+      # libraries test build platforms
+      testBuildPlatforms:
+      - Linux_x64
+      - windows_x64
 
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
@@ -116,6 +120,28 @@ jobs:
       collectionType: pmi
       collectionName: tests
 
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml
+    buildConfig: checked
+    platforms:
+    # Linux tests are built on the OSX machines.
+    # - OSX_x64
+    - Linux_arm
+    - Linux_arm64
+    - Linux_x64
+    - windows_x64
+    - windows_x86
+    - windows_arm64
+    - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64
+    helixQueueGroup: ci
+    helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+    jobParameters:
+      testGroup: outerloop
+      liveLibrariesBuildConfig: Release
+      collectionType: pmi
+      collectionName: tests_libraries
+
 - template: /eng/pipelines/common/platform-matrix.yml
   parameters:
     jobTemplate: /eng/pipelines/coreclr/templates/superpmi-job.yml

eng/pipelines/coreclr/templates/run-superpmi-job.yml

@@ -46,10 +46,10 @@ jobs:
       displayName: '${{ parameters.jobName }}'
 
     # tests collection takes longer so increase timeout to 8 hours
-    ${{ if eq(parameters.collectionName, 'tests') }}:
+    ${{ if or(eq(parameters.collectionName, 'tests'), eq(parameters.collectionName, 'tests_libraries')) }}:
       timeoutInMinutes: 480
-    ${{ if ne(parameters.collectionName, 'tests') }}:
-      timeoutInminutes: ${{ parameters.timeoutInMinutes }}
+    ${{ if and(ne(parameters.collectionName, 'tests'), ne(parameters.collectionName, 'tests_libraries')) }}:
+      timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
 
     variables:
 
@@ -101,6 +101,9 @@ jobs:
     - ${{ if eq(parameters.collectionName, 'tests') }}:
       - name: InputDirectory
         value: '$(managedTestArtifactRootFolderPath)'
+    - ${{ if eq(parameters.collectionName, 'tests_libraries') }}:
+      - name: InputDirectory
+        value: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)'
     workspace:
       clean: all
     pool:

eng/pipelines/coreclr/templates/superpmi-job.yml

@@ -90,7 +90,7 @@ jobs:
           artifactName: 'CoreCLRProduct__${{ parameters.runtimeVariant }}_$(osGroup)$(osSubgroup)_x64_$(buildConfig)'
           displayName: 'Coreclr product build (x64)'
 
-    # # Download and unzip managed test artifacts
+    # Download and unzip managed test artifacts
     - ${{ if eq(parameters.collectionName, 'tests') }}:
       - template: /eng/pipelines/common/download-artifact-step.yml
         parameters:
@@ -99,6 +99,23 @@ jobs:
           artifactName: '$(managedGenericTestArtifactName)'
           displayName: 'generic managed test artifacts'
 
+    # Download and unzip libraries test artifacts
+    - ${{ if eq(parameters.collectionName, 'tests_libraries') }}:
+      - template: /eng/pipelines/common/download-artifact-step.yml
+        parameters:
+          unpackFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)'
+          artifactFileName: 'libraries_test_assets_${{ parameters.osGroup }}_x64_Release$(archiveExtension)'
+          artifactName: ${{ format('libraries_test_assets_{0}_x64_Release', parameters.osGroup) }}
+          displayName: 'generic libraries test artifacts'
+
+    - task: ExtractFiles@1
+      displayName: 'Unzip Tests.zip'
+      inputs:
+        archiveFilePatterns: '$(Build.SourcesDirectory)/artifacts/tests/libraries_zipped/$(osGroup).$(archType).$(buildConfigUpper)/**/*.zip'
+        destinationFolder: '$(Build.SourcesDirectory)/artifacts/tests/libraries/$(osGroup).$(archType).$(buildConfigUpper)'
+        cleanDestinationFolder: true
+        overwriteExistingFiles: true
+
     # Create Core_Root
     - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) generatelayoutonly $(librariesOverrideArg)
       displayName: Create Core_Root

src/coreclr/ToolBox/superpmi/superpmi/parallelsuperpmi.cpp

@@ -539,6 +539,12 @@ int doParallelSuperPMI(CommandLine::Options& o)
                                       arrDiffMCListPath[i]);
         }
 
+        if (o.failureLimit > 0)
+        {
+            bytesWritten += sprintf_s(cmdLine + bytesWritten, MAX_CMDLINE_SIZE - bytesWritten, " -failureLimit %d",
+                                      o.failureLimit);
+        }
+
         bytesWritten += sprintf_s(cmdLine + bytesWritten, MAX_CMDLINE_SIZE - bytesWritten, " -v ewmin %s", spmiArgs);
 
         SECURITY_ATTRIBUTES sa;

src/coreclr/scripts/superpmi.py

@@ -200,6 +200,11 @@
 Only the final component can contain a `*` wildcard; the directory path cannot.
 """
 
+error_limit_help = """
+Specify the failure `limit` after which replay and asmdiffs will exit if it sees
+more than `limit` failures.
+"""
+
 # Start of parser object creation.
 
 parser = argparse.ArgumentParser(description=description)
@@ -247,6 +252,7 @@
 superpmi_common_parser.add_argument("-spmi_log_file", help=spmi_log_file_help)
 superpmi_common_parser.add_argument("-jit_name", help="Specify the filename of the jit to use, e.g., 'clrjit_win_arm64_x64.dll'. Default is clrjit.dll/libclrjit.so")
 superpmi_common_parser.add_argument("--altjit", action="store_true", help="Set the altjit variables on replay.")
+superpmi_common_parser.add_argument("-error_limit", help=error_limit_help)
 
 # subparser for collect
 collect_parser = subparsers.add_parser("collect", description=collect_description, parents=[core_root_parser, target_parser, superpmi_common_parser])
@@ -1639,6 +1645,9 @@ def replay(self):
             if self.coreclr_args.spmi_log_file is not None:
                 common_flags += [ "-w", self.coreclr_args.spmi_log_file ]
 
+            if self.coreclr_args.error_limit is not None:
+                common_flags += ["-failureLimit", self.coreclr_args.error_limit]
+
             if self.coreclr_args.jitoption:
                 for o in self.coreclr_args.jitoption:
                     repro_flags += "-jitoption", o
@@ -1867,6 +1876,9 @@ def replay_with_asm_diffs(self):
                     if self.coreclr_args.spmi_log_file is not None:
                         flags += [ "-w", self.coreclr_args.spmi_log_file ]
 
+                    if self.coreclr_args.error_limit is not None:
+                        flags += ["-failureLimit", self.coreclr_args.error_limit]
+
                     # Change the working directory to the Core_Root we will call SuperPMI from.
                     # This is done to allow libcoredistools to be loaded correctly on unix
                     # as the loadlibrary path will be relative to the current directory.
@@ -3256,6 +3268,13 @@ def setup_jit_path_arg(jit_path):
             return os.path.abspath(jit_path)
         return find_tool(coreclr_args, determine_jit_name(coreclr_args), search_path=False)  # It doesn't make sense to search PATH for the JIT dll.
 
+    def setup_error_limit(error_limit):
+        if error_limit is None:
+            return None
+        elif not error_limit.isnumeric():
+            return None
+        return error_limit
+
     def verify_jit_ee_version_arg():
 
         coreclr_args.verify(args,
@@ -3306,6 +3325,12 @@ def verify_superpmi_common_args():
                             lambda unused: True,
                             "Unable to set sequential.")
 
+        coreclr_args.verify(args,
+                            "error_limit",
+                            lambda unused: True,
+                            "Unable to set error_limit",
+                            modify_arg=setup_error_limit)
+
         coreclr_args.verify(args,
                             "spmi_log_file",
                             lambda unused: True,

src/coreclr/scripts/superpmi_setup.py

@@ -35,6 +35,7 @@
 
 import argparse
 import shutil
+import stat
 import subprocess
 import tempfile
 
@@ -304,13 +305,20 @@ def copy_directory(src_path, dst_path, verbose_output=True, match_func=lambda pa
         if os.path.isdir(src_item):
             copy_directory(src_item, dst_item, verbose_output, match_func)
         else:
-            if match_func(src_item):
+            try:
+                if match_func(src_item):
+                    if verbose_output:
+                        print("> copy {0} => {1}".format(src_item, dst_item))
+                    try:
+                        shutil.copy2(src_item, dst_item)
+                    except PermissionError as pe_error:
+                        print('Ignoring PermissionError: {0}'.format(pe_error))
+                else:
+                    if verbose_output:
+                        print("> skipping {0}".format(src_item))
+            except UnicodeEncodeError:
                 if verbose_output:
-                    print("> copy {0} => {1}".format(src_item, dst_item))
-                shutil.copy2(src_item, dst_item)
-            else:
-                if verbose_output:
-                    print("> skipping {0}".format(src_item))
+                    print("> Got UnicodeEncodeError")
 
 
 def copy_files(src_path, dst_path, file_names):
@@ -333,7 +341,10 @@ def copy_files(src_path, dst_path, file_names):
         dst_directory = path.dirname(dst_path_of_file)
         if not os.path.exists(dst_directory):
             os.makedirs(dst_directory)
-        shutil.copy2(f, dst_path_of_file)
+        try:
+            shutil.copy2(f, dst_path_of_file)
+        except PermissionError as pe_error:
+            print('Ignoring PermissionError: {0}'.format(pe_error))
 
 
 def partition_files(src_directory, dst_directory, max_size, exclude_directories=[],
@@ -381,7 +392,9 @@ def setup_microbenchmark(workitem_directory, arch):
             return
 
         run_command(
-            get_python_name() + [dotnet_install_script, "install", "--architecture", arch, "--install-dir", dotnet_directory, "--verbose"])
+            get_python_name() + [dotnet_install_script, "install", "--architecture", arch, "--install-dir",
+                                 dotnet_directory, "--verbose"])
+
 
 def get_python_name():
     """Gets the python name
@@ -441,12 +454,53 @@ def main(main_args):
     if is_windows:
         acceptable_copy = lambda path: any(path.endswith(extension) for extension in [".py", ".dll", ".exe", ".json"])
     else:
-        # Need to accept files without any extension, which is how executable filesnames look.
+        # Need to accept files without any extension, which is how executable file's names look.
         acceptable_copy = lambda path: (os.path.basename(path).find(".") == -1) or any(path.endswith(extension) for extension in [".py", ".dll", ".so", ".json"])
 
     print('Copying {} -> {}'.format(coreclr_args.core_root_directory, superpmi_dst_directory))
     copy_directory(coreclr_args.core_root_directory, superpmi_dst_directory, match_func=acceptable_copy)
 
+    # Copy all the test files to CORE_ROOT
+    # The reason is there are lot of dependencies with *.Tests.dll and to ensure we do not get
+    # Reflection errors, just copy everything to CORE_ROOT so for all individual partitions, the
+    # references will be present in CORE_ROOT.
+    if coreclr_args.collection_name == "tests_libraries":
+        print('Copying {} -> {}'.format(coreclr_args.input_directory, superpmi_dst_directory))
+
+        def make_readable(folder_name):
+            """Make file executable by changing the permission
+
+            Args:
+                folder_name (string): folder to mark with 744
+            """
+            if is_windows:
+                return
+
+            print("Inside make_readable")
+            run_command(["ls", "-l", folder_name])
+            for file_path, dirs, files in walk(folder_name, topdown=True):
+                for d in dirs:
+                    os.chmod(os.path.join(file_path, d),
+                    # read+write+execute for owner
+                    (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) |
+                    # read for group
+                    (stat.S_IRGRP) |
+                    # read for other
+                    (stat.S_IROTH))
+
+                for f in files:
+                    os.chmod(os.path.join(file_path, f),
+                    # read+write+execute for owner
+                    (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR) |
+                    # read for group
+                    (stat.S_IRGRP) |
+                    # read for other
+                    (stat.S_IROTH))
+            run_command(["ls", "-l", folder_name])
+
+        make_readable(coreclr_args.input_directory)
+        copy_directory(coreclr_args.input_directory, superpmi_dst_directory, match_func=acceptable_copy)
+
     # Workitem directories
     workitem_directory = path.join(source_directory, "workitem")
     input_artifacts = ""
@@ -497,8 +551,16 @@ def main(main_args):
         if coreclr_args.collection_type == "crossgen2":
             print('Adding exclusions for crossgen2')
             # Currently, trying to crossgen2 R2RTest\Microsoft.Build.dll causes a pop-up failure, so exclude it.
-            exclude_files += [ "Microsoft.Build.dll" ]
-        partition_files(coreclr_args.input_directory, input_artifacts, coreclr_args.max_size, exclude_directory, exclude_files)
+            exclude_files += ["Microsoft.Build.dll"]
+
+        if coreclr_args.collection_name == "tests_libraries":
+            # tests_libraries artifacts contains files from core_root folder. Exclude them.
+            core_root_dir = coreclr_args.core_root_directory
+            exclude_files += [item for item in os.listdir(core_root_dir)
+                              if isfile(join(core_root_dir, item)) and (item.endswith(".dll") or item.endswith(".exe"))]
+
+        partition_files(coreclr_args.input_directory, input_artifacts, coreclr_args.max_size, exclude_directory,
+                        exclude_files)
 
     # Set variables
     print('Setting pipeline variables:')