From 332f118d15b75af94963747ae1610de27c0aacaf Mon Sep 17 00:00:00 2001 From: David Wrighton Date: Fri, 2 Oct 2020 13:35:46 -0700 Subject: [PATCH] Crossgen2 cross target testing (#42663) Add cross compilation testing for crossgen2. The goal here is to cover cross compilation for the framework libraries, with support for testing cross compilation behavior between across the large swath of possible cross target possibilities. This testing is added to the crossgen2 outerloop and is based on the work that @echesakovMSFT built for crossgen1 for cross target support for Linux X64 to Linux Arm. (The testing model has been tweaked to allow for general purpose cross target testing.) Important details. There is now a cross targeting build job, which builds (on either X64 Windows or X64 Linux) the framework dlls for a given target architecture, and captures their SHA1 hashes. Then there is a comparison job which will run crossgen2 on an arbitrary set of platforms targetting a specific OS/Architecture pair. The current state is that is known that the x86 compiler does not quite produce identical output when varied between 64 and 32 bit, there are significant issues compiling arm32 code, and arm64 code is nearly perfect with cross compilation. --- eng/pipelines/coreclr/crossgen2-outerloop.yml | 131 ++- .../crossgen2-comparison-build-job.yml | 169 ++++ .../templates/crossgen2-comparison-job.yml | 176 ++++ .../Common/scripts/crossgen2_comparison.py | 891 ++++++++++++++++++ 4 files changed, 1366 insertions(+), 1 deletion(-) create mode 100644 eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml create mode 100644 eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml create mode 100644 src/tests/Common/scripts/crossgen2_comparison.py diff --git a/eng/pipelines/coreclr/crossgen2-outerloop.yml b/eng/pipelines/coreclr/crossgen2-outerloop.yml index 505e3ecc2d1dee..9ad74c254afad6 100644 --- a/eng/pipelines/coreclr/crossgen2-outerloop.yml +++ b/eng/pipelines/coreclr/crossgen2-outerloop.yml @@ -18,7 +18,7 @@ jobs: - template: /eng/pipelines/common/platform-matrix.yml parameters: - jobTemplate: /eng/pipelines/common/build-coreclr-and-libraries-job.yml + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml buildConfig: checked platforms: - Linux_x64 @@ -30,6 +30,36 @@ jobs: jobParameters: testGroup: outerloop +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/build-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + jobParameters: + testGroup: outerloop + +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/libraries/build-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - OSX_x64 + - Windows_NT_x86 + - Windows_NT_x64 + - Windows_NT_arm64 + - CoreClrTestBuildHost # Either OSX_x64 or Linux_x64 + jobParameters: + isOfficialBuild: false + liveRuntimeBuildConfig: Release + - template: /eng/pipelines/common/platform-matrix.yml parameters: jobTemplate: /eng/pipelines/common/templates/runtimes/build-test-job.yml @@ -74,3 +104,102 @@ jobs: crossgen2: true displayNameArgs: R2R_CG2 liveLibrariesBuildConfig: Release + +# Build Crossgen2 baselines +# These are the various crossgen2 targets that are supported, and cover all major +# significantly different code generators +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml + buildConfig: Release + platforms: + - Linux_arm + - Linux_arm64 + - Linux_x64 + - Windows_NT_x86 + - Windows_NT_x64 + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + +# test crossgen target Windows X86 +# This job verifies that 32-bit and 64 bit crossgen2 produces the same binaries, +# and that cross-os targetting works +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml + buildConfig: Release + platforms: + - Linux_x64 + - Windows_NT_x86 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + targetos: Windows_NT + targetarch: x86 + +# test target Linux X64 +# verify that cross OS targetting works +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml + buildConfig: Release + platforms: + - Windows_NT_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + targetos: Linux + targetarch: x64 + +# test target Windows X64 +# verify that cross OS targetting works +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml + buildConfig: Release + platforms: + - Linux_x64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + targetos: Windows_NT + targetarch: x64 + +# test target Linux arm +# verify that cross architecture targetting works +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml + buildConfig: Release + platforms: + - Linux_arm + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + targetos: Linux + targetarch: arm + +# test target Linux arm64 +# verify that cross architecture targetting works +- template: /eng/pipelines/common/platform-matrix.yml + parameters: + jobTemplate: /eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml + buildConfig: Release + platforms: + - Linux_arm64 + helixQueueGroup: pr + helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml + jobParameters: + testGroup: outerloop + liveLibrariesBuildConfig: Release + targetos: Linux + targetarch: arm64 diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml new file mode 100644 index 00000000000000..553131f6259199 --- /dev/null +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-build-job.yml @@ -0,0 +1,169 @@ +parameters: + buildConfig: '' + archType: '' + osGroup: '' + osSubgroup: '' + container: '' + helixQueues: '' + runtimeVariant: '' + crossrootfsDir: '' + stagedBuild: false + variables: {} + pool: '' + + # When set to a non-empty value (Debug / Release), it determines libraries + # build configuration to use for the tests. Setting this property implies + # a dependency of this job on the appropriate libraries build and is used + # to construct the name of the Azure artifact representing libraries build + # to use for building the tests. + liveLibrariesBuildConfig: '' + +### Crossgen-comparison build job +### +### Ensure that the output of cross-architecture, e.g. x64-hosted-arm-targeting, +### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. + +jobs: +- template: xplat-pipeline-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + stagedBuild: ${{ parameters.stagedBuild }} + runtimeVariant: ${{ parameters.runtimeVariant }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + helixType: 'test/crossgen-comparison/' + pool: ${{ parameters.pool }} + + # Compute job name from template parameters + name: ${{ format('test_crossgen2_comparison_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + displayName: ${{ format('Test crossgen2-comparison build {0}{1} {2} {3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + + variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-HelixApi-Access + - name: hostArchType + value: x64 + - name: targetFlavor + value: $(osGroup).$(archType).$(buildConfigUpper) + - name: crossFlavor + value: $(osGroup).$(hostArchType)_$(archType).$(buildConfigUpper) + - name: artifactsDirectory + value: $(Build.SourcesDirectory)$(dir)artifacts + - name: binDirectory + value: $(artifactsDirectory)$(dir)bin + - name: productDirectory + value: $(binDirectory)$(dir)coreclr + - name: workItemDirectory + value: $(artifactsDirectory)$(dir)tests$(dir)coreclr$(dir)$(targetFlavor)$(dir)Tests$(dir)CrossCompileRoot + - name: crossgencompare_build_artifact + value: crossgen_comparison_build_$(osGroup)$(osSubgroup)_$(archType) + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - name: target_crossgen2_os + value: windows + - ${{ if eq(parameters.osGroup, 'Linux') }}: + - name: target_crossgen2_os + value: linux + - ${{ if eq(parameters.osGroup, 'OSX') }}: + - name: target_crossgen2_os + value: osx + - name: crossgen2location + value: $(productDirectory)$(dir)$(targetFlavor)$(dir)crossgen2$(dir)crossgen2.dll + - ${{ if ne(parameters.archType, 'x64') }}: + - name: crossgen2location + value: $(productDirectory)$(dir)$(targetFlavor)$(dir)x64$(dir)crossgen2$(dir)crossgen2.dll + - name: librariesProductDllDir + value: $(Build.SourcesDirectory)$(dir)artifacts$(dir)bin$(dir)runtime$(dir)net5.0-$(osGroup)$(osSubgroup)-$(buildConfig)-$(archType) + + - ${{ parameters.variables }} + + # Test job depends on the corresponding build job + dependsOn: + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + + # Run all steps in the container. + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} + timeoutInMinutes: 180 # 3 hrs + + steps: + + # Download product build + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' + displayName: 'product build' + + # Optionally download live-built libraries + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' + artifactName: '$(librariesBuildArtifactName)' + displayName: 'live-built libraries' + + # Populate Core_Root + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly + displayName: Populate Core_Root + + # Create work item directory and populate with assemblies + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + - script: | + mkdir -p $(workItemDirectory) + mkdir -p $(workItemDirectory)/log + mkdir -p $(workItemDirectory)/dlls + cp $(librariesProductDllDir)/* $(workItemDirectory)/dlls + cp $(productDirectory)/$(targetFlavor)/IL/System.Private.CoreLib.dll $(workItemDirectory)/dlls + displayName: Create directories + - ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + - script: | + md $(workItemDirectory)\log + md $(workItemDirectory)\dlls + echo copy $(librariesProductDllDir)\* $(workItemDirectory)\dlls + copy $(librariesProductDllDir)\* $(workItemDirectory)\dlls + echo copy $(productDirectory)\$(targetFlavor)\IL\System.Private.CoreLib.dll $(workItemDirectory)\dlls + copy $(productDirectory)\$(targetFlavor)\IL\System.Private.CoreLib.dll $(workItemDirectory)\dlls + displayName: Create directories + + # Create baseline output on the host (x64) machine + - task: PythonScript@0 + displayName: Create cross-platform crossgen baseline + inputs: + scriptSource: 'filePath' + scriptPath: $(Build.SourcesDirectory)/src/tests/Common/scripts/crossgen2_comparison.py + ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + pythonInterpreter: /usr/bin/python3 + arguments: + crossgen_framework + --crossgen $(crossgen2location) + --dotnet $(Build.SourcesDirectory)/dotnet.sh + --core_root $(workItemDirectory)/dlls + --result_dir $(workItemDirectory)/log + --target_os $(target_crossgen2_os) + --target_arch $(archType) + ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + arguments: + crossgen_framework + --crossgen $(crossgen2location) + --dotnet $(Build.SourcesDirectory)\dotnet.cmd + --core_root $(workItemDirectory)\dlls + --result_dir $(workItemDirectory)\log + --target_os $(target_crossgen2_os) + --target_arch $(archType) + + - task: PublishPipelineArtifact@1 + displayName: Publish cross compiled component + inputs: + targetPath: $(workItemDirectory) + artifactName: $(crossgencompare_build_artifact) + continueOnError: true + condition: always() \ No newline at end of file diff --git a/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml new file mode 100644 index 00000000000000..65aed434a11aaa --- /dev/null +++ b/eng/pipelines/coreclr/templates/crossgen2-comparison-job.yml @@ -0,0 +1,176 @@ +parameters: + buildConfig: '' + archType: '' + osGroup: '' + osSubgroup: '' + container: '' + helixQueues: '' + runtimeVariant: '' + crossrootfsDir: '' + stagedBuild: false + variables: {} + pool: '' + targetarch: '' + targetos: '' + + # When set to a non-empty value (Debug / Release), it determines libraries + # build configuration to use for the tests. Setting this property implies + # a dependency of this job on the appropriate libraries build and is used + # to construct the name of the Azure artifact representing libraries build + # to use for building the tests. + liveLibrariesBuildConfig: '' + +### Crossgen-comparison job +### +### Ensure that the output of cross-architecture, e.g. x64-hosted-arm-targeting, +### crossgen matches that of native, e.g. arm-hosted-arm-targeting, crossgen. + +jobs: +- template: xplat-pipeline-job.yml + parameters: + buildConfig: ${{ parameters.buildConfig }} + archType: ${{ parameters.archType }} + osGroup: ${{ parameters.osGroup }} + osSubgroup: ${{ parameters.osSubgroup }} + stagedBuild: ${{ parameters.stagedBuild }} + runtimeVariant: ${{ parameters.runtimeVariant }} + liveLibrariesBuildConfig: ${{ parameters.liveLibrariesBuildConfig }} + helixType: 'test/crossgen-comparison/' + pool: ${{ parameters.pool }} + targetos: ${{ parameters.targetos }} + targetarch: ${{ parameters.targetarch }} + + # Compute job name from template parameters + name: ${{ format('test_crossgen2_comparison_{0}{1}_{2}_{3}_{4}_{5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.targetarch, parameters.targetos) }} + displayName: ${{ format('Test crossgen2-comparison {0}{1} {2} {3} to {4} {5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.targetarch, parameters.targetos) }} + + crossrootfsDir: ${{ parameters.crossrootfsDir }} + + variables: + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNet-HelixApi-Access + - name: hostArchType + value: x64 + - name: targetFlavor + value: $(osGroup).$(archType).$(buildConfigUpper) + - name: artifactsDirectory + value: $(Build.SourcesDirectory)$(dir)artifacts + - name: binDirectory + value: $(artifactsDirectory)$(dir)bin + - name: productDirectory + value: $(binDirectory)$(dir)coreclr + - name: workItemDirectory + value: $(artifactsDirectory)$(dir)tests$(dir)coreclr$(dir)$(targetFlavor)$(dir)Tests$(dir)Core_Root + - name: targetarch + value: ${{ parameters.targetarch }} + - name: crossgencompare_build_artifact + value: crossgen_comparison_build_${{ parameters.targetos }}_${{ parameters.targetarch }} + - ${{ if eq(parameters.targetos, 'Windows_NT') }}: + - name: target_crossgen2_os + value: windows + - ${{ if eq(parameters.targetos, 'Linux') }}: + - name: target_crossgen2_os + value: linux + - ${{ if eq(parameters.targetos, 'OSX') }}: + - name: target_crossgen2_os + value: osx + + - ${{ parameters.variables }} + + # Test job depends on the corresponding build job + dependsOn: + - ${{ format('test_crossgen2_comparison_build_{0}_{1}_Release', parameters.targetos, parameters.targetarch)}} + - ${{ format('coreclr_{0}_product_build_{1}{2}_{3}_{4}', parameters.runtimeVariant, parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig) }} + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - ${{ format('libraries_build_{0}{1}_{2}_{3}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.liveLibrariesBuildConfig) }} + + # Run all steps in the container. + # Note that the containers are defined in platform-matrix.yml + container: ${{ parameters.container }} + timeoutInMinutes: 180 # 3 hrs + + steps: + + # Download product build + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(buildProductRootFolderPath) + artifactFileName: '$(buildProductArtifactName)$(archiveExtension)' + artifactName: '$(buildProductArtifactName)' + displayName: 'product build' + + # Optionally download live-built libraries + - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}: + - template: /eng/pipelines/common/download-artifact-step.yml + parameters: + unpackFolder: $(librariesDownloadDir) + cleanUnpackFolder: false + artifactFileName: '$(librariesBuildArtifactName)$(archiveExtension)' + artifactName: '$(librariesBuildArtifactName)' + displayName: 'live-built libraries' + + # Populate Core_Root + - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) $(buildConfig) $(archType) $(crossArg) generatelayoutonly + displayName: Populate Core_Root + + - task: DownloadPipelineArtifact@2 + displayName: Download Preprepared crossgen inputs and expected values + inputs: + artifact: '$(crossgencompare_build_artifact)' + path: '$(workItemDirectory)$(dir)prebuiltWork' + + # Send payload to Helix where the native output is generated and compared to the baseline + - template: /eng/common/templates/steps/send-to-helix.yml + parameters: + DisplayNamePrefix: Run native crossgen and compare output to baseline + osGroup: ${{ parameters.osGroup }} + HelixSource: $(_HelixSource) + HelixType: 'test/crossgen-comparison/' + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + HelixAccessToken: $(HelixApiAccessToken) + HelixTargetQueues: ${{ join(' ', parameters.helixQueues) }} + ${{ if ne(variables['System.TeamProject'], 'internal') }}: + Creator: $(Creator) + WorkItemTimeout: 3:00 # 3 hours + WorkItemDirectory: '$(workItemDirectory)' + CorrelationPayloadDirectory: '$(Build.SourcesDirectory)/src/tests/Common/scripts' + ${{ if ne(parameters.osGroup, 'Windows_NT') }}: + WorkItemCommand: + echo Targeting $(targetFlavor) ; + chmod +x $HELIX_WORKITEM_PAYLOAD/corerun; + mkdir -p $HELIX_WORKITEM_PAYLOAD/log; + export CORE_ROOT=$HELIX_CORRELATION_PAYLOAD; + python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py crossgen_framework + --crossgen $HELIX_WORKITEM_PAYLOAD/crossgen2/crossgen2.dll + --dotnet $HELIX_WORKITEM_PAYLOAD/corerun + --core_root $HELIX_WORKITEM_PAYLOAD/prebuiltWork/dlls + --result_dir $HELIX_WORKITEM_PAYLOAD/log + --target_os $(target_crossgen2_os) + --target_arch $(targetarch); + python3 -u $HELIX_CORRELATION_PAYLOAD/crossgen2_comparison.py compare + --base_dir $HELIX_WORKITEM_PAYLOAD/prebuiltWork/log + --diff_dir $HELIX_WORKITEM_PAYLOAD/log + ${{ if eq(parameters.osGroup, 'Windows_NT') }}: + WorkItemCommand: + echo Targeting $(targetFlavor) & + md %HELIX_WORKITEM_PAYLOAD%\log & + set CORE_ROOT=%HELIX_CORRELATION_PAYLOAD% & + python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py crossgen_framework + --crossgen %HELIX_WORKITEM_PAYLOAD%\crossgen2\crossgen2.dll + --dotnet %HELIX_WORKITEM_PAYLOAD%\corerun.exe + --core_root %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\dlls + --result_dir %HELIX_WORKITEM_PAYLOAD%\log + --target_os $(target_crossgen2_os) + --target_arch $(targetarch) & + python -u %HELIX_CORRELATION_PAYLOAD%\crossgen2_comparison.py compare + --base_dir %HELIX_WORKITEM_PAYLOAD%\prebuiltWork\log + --diff_dir %HELIX_WORKITEM_PAYLOAD%\log + + # Publish log + - task: PublishPipelineArtifact@1 + displayName: Publish log + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/log + artifactName: ${{ format('Testlog_crossgen2_comparison_{0}{1}_{2}_{3}_{4}_{5}', parameters.osGroup, parameters.osSubgroup, parameters.archType, parameters.buildConfig, parameters.targetarch, parameters.targetos) }} + continueOnError: true + condition: always() diff --git a/src/tests/Common/scripts/crossgen2_comparison.py b/src/tests/Common/scripts/crossgen2_comparison.py new file mode 100644 index 00000000000000..e3db1300bea118 --- /dev/null +++ b/src/tests/Common/scripts/crossgen2_comparison.py @@ -0,0 +1,891 @@ +#!/usr/bin/env python +# +# Licensed to the .NET Foundation under one or more agreements. +# The .NET Foundation licenses this file to you under the MIT license. +# +################################################################################ +# +# Module: crossgen_comparison.py +# +# Notes: +# +# Script that +# 1) runs crossgen on System.Private.CoreLib.dll and CoreFX assemblies and +# collects information about the crossgen behaviour (such as the return code, +# stdout/stderr streams, SHA256 hash sum of the resulting file). +# 2) compares the collected information from two crossgen scenarios (e.g. +# x86_arm vs. arm_arm) and report all the differences in their behaviour +# (such as mismatches in the resulting files; hash sums, or missing files). +# +# Example: +# +# The following command +# +# ~/git/coreclr$ python tests/scripts/crossgen_comparison.py crossgen_corelib +# --crossgen artifacts/bin/coreclr/Linux.arm.Checked/crossgen +# --il_corelib artifacts/bin/coreclr/Linux.arm.Checked/IL/System.Private.CoreLib.dll +# --result_dir Linux.arm_arm.Checked +# +# runs Hostarm/arm crossgen on System.Private.CoreLib.dll and puts all the +# information in files Linux.arm_arm.Checked/System.Private.CoreLib-NativeOrReadyToRunImage.json +# and System.Private.CoreLib-DebuggingFile.json +# +# ~/git/coreclr$ cat Linux.arm_arm.Checked/System.Private.CoreLib-NativeOrReadyToRunImage.json +# { +# "AssemblyName": "System.Private.CoreLib", +# "ReturnCode": 0, +# "OutputFileHash": "4d27c7f694c20974945e4f7cb43263286a18c56f4d00aac09f6124caa372ba0a", +# "StdErr": [], +# "StdOut": [ +# "Native image /tmp/tmp9ZX7gl/System.Private.CoreLib.dll generated successfully." +# ], +# "OutputFileType": "NativeOrReadyToRunImage", +# "OutputFileSizeInBytes": 9564160 +# } +# +# ~/git/coreclr$ cat Linux.x64_arm.Checked/System.Private.CoreLib-DebuggingFile.json +# { +# "ReturnCode": 0, +# "StdOut": [ +# "Successfully generated perfmap for native assembly '/tmp/tmp9ZX7gl/System.Private.CoreLib.dll'." +# ], +# "OutputFileHash": "f4fff0d88193d3a1422f9f0806a6cea6ac6c1aab0499968c183cbb0755e1084b", +# "OutputFileType": "DebuggingFile", +# "StdErr": [], +# "OutputFileSizeInBytes": 1827867, +# "AssemblyName": "System.Private.CoreLib" +# } +# +# The following command +# +# ~/git/coreclr$ python tests/scripts/crossgen_comparison.py crossgen_dotnet_sdk +# --crossgen artifacts/bin/coreclr/Linux.arm.Checked/x64/crossgen +# --il_corelib artifacts/bin/coreclr/Linux.arm.Checked/IL/System.Private.CoreLib.dll +# --dotnet_sdk dotnet-sdk-latest-linux-arm.tar.gz +# --result_dir Linux.x64_arm.Checked +# +# runs Hostx64/arm crossgen on System.Private.CoreLib.dll in artifacts/Product and on +# all the assemblies inside dotnet-sdk-latest-linux-arm.tar.gz and stores the +# collected information in directory Linux.x64_arm.Checked +# +# ~/git/coreclr$ ls Linux.x64_arm.Checked | head +# Microsoft.AI.DependencyCollector.dll.json +# Microsoft.ApplicationInsights.AspNetCore.dll.json +# Microsoft.ApplicationInsights.dll.json +# Microsoft.AspNetCore.Antiforgery.dll.json +# Microsoft.AspNetCore.ApplicationInsights.HostingStartup.dll.json +# Microsoft.AspNetCore.Authentication.Abstractions.dll.json +# Microsoft.AspNetCore.Authentication.Cookies.dll.json +# Microsoft.AspNetCore.Authentication.Core.dll.json +# Microsoft.AspNetCore.Authentication.dll.json +# Microsoft.AspNetCore.Authentication.Facebook.dll.json +# +# The following command +# +# ~/git/coreclr$ python -u tests/scripts/crossgen_comparison.py compare +# --base_dir Linux.x64_arm.Checked +# --diff_dir Linux.x86_arm.Checked +# +# compares the results of Hostx64/arm crossgen and Hostx86/arm crossgen. +################################################################################ +################################################################################ + +import argparse +import datetime +import asyncio +import glob +import json +import hashlib +import multiprocessing +import os +import tarfile +import tempfile +import re +import shutil +import subprocess +import sys + +################################################################################ +# Argument Parser +################################################################################ + +def build_argument_parser(): + description = """Script that runs crossgen on different assemblies and + collects/compares information about the crossgen behaviour.""" + + parser = argparse.ArgumentParser(description=description) + + subparsers = parser.add_subparsers() + + crossgen_corelib_description = """Runs crossgen on IL System.Private.CoreLib.dll and + collects information about the run.""" + + corelib_parser = subparsers.add_parser('crossgen_corelib', description=crossgen_corelib_description) + corelib_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True) + corelib_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) + corelib_parser.add_argument('--result_dir', dest='result_dirname', required=True) + corelib_parser.set_defaults(func=crossgen_corelib) + + framework_parser_description = """Runs crossgen on each assembly in Core_Root and + collects information about all the runs.""" + + framework_parser = subparsers.add_parser('crossgen_framework', description=framework_parser_description) + framework_parser.add_argument('--dotnet', dest='dotnet', required=True) + framework_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True) + framework_parser.add_argument('--target_os', dest='target_os', required=True) + framework_parser.add_argument('--target_arch', dest='target_arch', required=True) + framework_parser.add_argument('--core_root', dest='core_root', required=True) + framework_parser.add_argument('--result_dir', dest='result_dirname', required=True) + framework_parser.set_defaults(func=crossgen_framework) + + + dotnet_sdk_parser_description = "Unpack .NET SDK archive file and runs crossgen on each assembly." + dotnet_sdk_parser = subparsers.add_parser('crossgen_dotnet_sdk', description=dotnet_sdk_parser_description) + dotnet_sdk_parser.add_argument('--crossgen', dest='crossgen_executable_filename', required=True) + dotnet_sdk_parser.add_argument('--il_corelib', dest='il_corelib_filename', required=True) + dotnet_sdk_parser.add_argument('--dotnet_sdk', dest='dotnet_sdk_filename', required=True) + dotnet_sdk_parser.add_argument('--result_dir', dest='result_dirname', required=True) + dotnet_sdk_parser.set_defaults(func=crossgen_dotnet_sdk) + + compare_parser_description = "Compares crossgen results in directories {base_dir} and {diff_dir}" + + compare_parser = subparsers.add_parser('compare', description=compare_parser_description) + compare_parser.add_argument('--base_dir', dest='base_dirname', required=True) + compare_parser.add_argument('--diff_dir', dest='diff_dirname', required=True) + compare_parser.set_defaults(func=compare_results) + + return parser + +################################################################################ +# Helper class +################################################################################ + +class AsyncSubprocessHelper: + def __init__(self, items, subproc_count=multiprocessing.cpu_count(), verbose=False): + item_queue = asyncio.Queue() + for item in items: + item_queue.put_nowait(item) + + self.items = items + self.subproc_count = subproc_count + self.verbose = verbose + + if 'win32' in sys.platform: + # Windows specific event-loop policy & cmd + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + + async def __get_item__(self, item, index, size, async_callback, *extra_args): + """ Wrapper to the async callback which will schedule based on the queue + """ + + # Wait for the queue to become free. Then start + # running the sub process. + subproc_id = await self.subproc_count_queue.get() + + print_prefix = "" + + if self.verbose: + print_prefix = "[{}:{}]: ".format(index, size) + + await async_callback(print_prefix, item, *extra_args) + + # Add back to the queue, incase another process wants to run. + self.subproc_count_queue.put_nowait(subproc_id) + + async def __run_to_completion__(self, async_callback, *extra_args): + """ async wrapper for run_to_completion + """ + + chunk_size = self.subproc_count + + # Create a queue with a chunk size of the cpu count + # + # Each run_crossgen invocation will remove an item from the + # queue before running a potentially long running pmi run. + # + # When the queue is drained, we will wait queue.get which + # will wait for when a run_crossgen instance has added back to the + subproc_count_queue = asyncio.Queue(chunk_size) + diff_queue = asyncio.Queue() + + for item in self.items: + diff_queue.put_nowait(item) + + for item in range(chunk_size): + subproc_count_queue.put_nowait(item) + + self.subproc_count_queue = subproc_count_queue + tasks = [] + size = diff_queue.qsize() + + count = 1 + item = diff_queue.get_nowait() if not diff_queue.empty() else None + while item is not None: + tasks.append(self.__get_item__(item, count, size, async_callback, *extra_args)) + count += 1 + + item = diff_queue.get_nowait() if not diff_queue.empty() else None + + await asyncio.gather(*tasks) + + def run_to_completion(self, async_callback, *extra_args): + """ Run until the item queue has been depleted + + Notes: + Acts as a wrapper to abstract the async calls to + async_callback. Note that this will allow cpu_count + amount of running subprocesses. Each time the queue + is emptied, another process will start. Note that + the python code is single threaded, it will just + rely on async/await to start subprocesses at + subprocess_count + """ + + reset_env = os.environ.copy() + + loop = asyncio.get_event_loop() + loop.run_until_complete(self.__run_to_completion__(async_callback, *extra_args)) + loop.close() + + os.environ.update(reset_env) + +################################################################################ +# Globals +################################################################################ + +g_frameworkcompile_failed = False + +# List of framework assemblies used for crossgen_framework command +g_Framework_Assemblies = [ + 'Microsoft.Bcl.AsyncInterfaces.dll', + 'Microsoft.CSharp.dll', + 'Microsoft.Extensions.Caching.Abstractions.dll', + 'Microsoft.Extensions.Caching.Memory.dll', + 'Microsoft.Extensions.Configuration.Abstractions.dll', + 'Microsoft.Extensions.Configuration.Binder.dll', + 'Microsoft.Extensions.Configuration.CommandLine.dll', + 'Microsoft.Extensions.Configuration.dll', + 'Microsoft.Extensions.Configuration.EnvironmentVariables.dll', + 'Microsoft.Extensions.Configuration.FileExtensions.dll', + 'Microsoft.Extensions.Configuration.Ini.dll', + 'Microsoft.Extensions.Configuration.Json.dll', + 'Microsoft.Extensions.Configuration.UserSecrets.dll', + 'Microsoft.Extensions.Configuration.Xml.dll', + 'Microsoft.Extensions.DependencyInjection.Abstractions.dll', + 'Microsoft.Extensions.DependencyInjection.dll', + 'Microsoft.Extensions.DependencyModel.dll', + 'Microsoft.Extensions.FileProviders.Abstractions.dll', + 'Microsoft.Extensions.FileProviders.Composite.dll', + 'Microsoft.Extensions.FileProviders.Physical.dll', + 'Microsoft.Extensions.FileSystemGlobbing.dll', + 'Microsoft.Extensions.Hosting.Abstractions.dll', + 'Microsoft.Extensions.Hosting.dll', + 'Microsoft.Extensions.Http.dll', + 'Microsoft.Extensions.Logging.Abstractions.dll', + 'Microsoft.Extensions.Logging.Configuration.dll', + 'Microsoft.Extensions.Logging.Console.dll', + 'Microsoft.Extensions.Logging.Debug.dll', + 'Microsoft.Extensions.Logging.dll', + 'Microsoft.Extensions.Logging.EventLog.dll', + 'Microsoft.Extensions.Logging.EventSource.dll', + 'Microsoft.Extensions.Logging.TraceSource.dll', + 'Microsoft.Extensions.Options.ConfigurationExtensions.dll', + 'Microsoft.Extensions.Options.DataAnnotations.dll', + 'Microsoft.Extensions.Options.dll', + 'Microsoft.Extensions.Primitives.dll', + 'Microsoft.VisualBasic.Core.dll', + 'Microsoft.VisualBasic.dll', + 'Microsoft.Win32.Primitives.dll', + 'Microsoft.Win32.Registry.AccessControl.dll', + 'Microsoft.Win32.Registry.dll', + 'Microsoft.Win32.SystemEvents.dll', + 'mscorlib.dll', + 'netstandard.dll', + 'System.AppContext.dll', + 'System.Buffers.dll', + 'System.CodeDom.dll', + 'System.Collections.Concurrent.dll', + 'System.Collections.dll', + 'System.Collections.Immutable.dll', + 'System.Collections.NonGeneric.dll', + 'System.Collections.Specialized.dll', + 'System.ComponentModel.Annotations.dll', + 'System.ComponentModel.Composition.dll', + 'System.ComponentModel.Composition.Registration.dll', + 'System.ComponentModel.DataAnnotations.dll', + 'System.ComponentModel.dll', + 'System.ComponentModel.EventBasedAsync.dll', + 'System.ComponentModel.Primitives.dll', + 'System.ComponentModel.TypeConverter.dll', + 'System.Composition.AttributedModel.dll', + 'System.Composition.Convention.dll', + 'System.Composition.Hosting.dll', + 'System.Composition.Runtime.dll', + 'System.Composition.TypedParts.dll', + 'System.Configuration.ConfigurationManager.dll', + 'System.Configuration.dll', + 'System.Console.dll', + 'System.Core.dll', + 'System.Data.Common.dll', + 'System.Data.DataSetExtensions.dll', + 'System.Data.dll', + 'System.Data.Odbc.dll', + 'System.Data.OleDb.dll', + 'System.Diagnostics.Contracts.dll', + 'System.Diagnostics.Debug.dll', + 'System.Diagnostics.DiagnosticSource.dll', + 'System.Diagnostics.EventLog.dll', + 'System.Diagnostics.FileVersionInfo.dll', + 'System.Diagnostics.PerformanceCounter.dll', + 'System.Diagnostics.Process.dll', + 'System.Diagnostics.StackTrace.dll', + 'System.Diagnostics.TextWriterTraceListener.dll', + 'System.Diagnostics.Tools.dll', + 'System.Diagnostics.TraceSource.dll', + 'System.Diagnostics.Tracing.dll', + 'System.DirectoryServices.AccountManagement.dll', + 'System.DirectoryServices.dll', + 'System.DirectoryServices.Protocols.dll', + 'System.dll', + 'System.Drawing.Common.dll', + 'System.Drawing.dll', + 'System.Drawing.Primitives.dll', + 'System.Dynamic.Runtime.dll', + 'System.Formats.Asn1.dll', + 'System.Formats.Cbor.dll', + 'System.Globalization.Calendars.dll', + 'System.Globalization.dll', + 'System.Globalization.Extensions.dll', + 'System.IO.Compression.Brotli.dll', + 'System.IO.Compression.dll', + 'System.IO.Compression.FileSystem.dll', + 'System.IO.Compression.ZipFile.dll', + 'System.IO.dll', + 'System.IO.FileSystem.AccessControl.dll', + 'System.IO.FileSystem.dll', + 'System.IO.FileSystem.DriveInfo.dll', + 'System.IO.FileSystem.Primitives.dll', + 'System.IO.FileSystem.Watcher.dll', + 'System.IO.IsolatedStorage.dll', + 'System.IO.MemoryMappedFiles.dll', + 'System.IO.Packaging.dll', + 'System.IO.Pipelines.dll', + 'System.IO.Pipes.AccessControl.dll', + 'System.IO.Pipes.dll', + 'System.IO.Ports.dll', + 'System.IO.UnmanagedMemoryStream.dll', + 'System.Linq.dll', + 'System.Linq.Expressions.dll', + 'System.Linq.Parallel.dll', + 'System.Linq.Queryable.dll', + 'System.Management.dll', + 'System.Memory.dll', + 'System.Net.Connections.dll', + 'System.Net.dll', + 'System.Net.Http.dll', + 'System.Net.Http.Json.dll', + 'System.Net.Http.WinHttpHandler.dll', + 'System.Net.HttpListener.dll', + 'System.Net.Mail.dll', + 'System.Net.NameResolution.dll', + 'System.Net.NetworkInformation.dll', + 'System.Net.Ping.dll', + 'System.Net.Primitives.dll', + 'System.Net.Requests.dll', + 'System.Net.Security.dll', + 'System.Net.ServicePoint.dll', + 'System.Net.Sockets.dll', + 'System.Net.WebClient.dll', + 'System.Net.WebHeaderCollection.dll', + 'System.Net.WebProxy.dll', + 'System.Net.WebSockets.Client.dll', + 'System.Net.WebSockets.dll', + 'System.Net.WebSockets.WebSocketProtocol.dll', + 'System.Numerics.dll', + 'System.Numerics.Tensors.dll', + 'System.Numerics.Vectors.dll', + 'System.ObjectModel.dll', + 'System.Private.CoreLib.dll', + 'System.Private.DataContractSerialization.dll', + 'System.Private.Uri.dll', + 'System.Private.Xml.dll', + 'System.Private.Xml.Linq.dll', + 'System.Reflection.Context.dll', + 'System.Reflection.DispatchProxy.dll', + 'System.Reflection.dll', + 'System.Reflection.Emit.dll', + 'System.Reflection.Emit.ILGeneration.dll', + 'System.Reflection.Emit.Lightweight.dll', + 'System.Reflection.Extensions.dll', + 'System.Reflection.Metadata.dll', + 'System.Reflection.MetadataLoadContext.dll', + 'System.Reflection.Primitives.dll', + 'System.Reflection.TypeExtensions.dll', + 'System.Resources.Extensions.dll', + 'System.Resources.Reader.dll', + 'System.Resources.ResourceManager.dll', + 'System.Resources.Writer.dll', + 'System.Runtime.Caching.dll', + 'System.Runtime.CompilerServices.Unsafe.dll', + 'System.Runtime.CompilerServices.VisualC.dll', + 'System.Runtime.dll', + 'System.Runtime.Extensions.dll', + 'System.Runtime.Handles.dll', + 'System.Runtime.InteropServices.dll', + 'System.Runtime.InteropServices.RuntimeInformation.dll', + 'System.Runtime.Intrinsics.dll', + 'System.Runtime.Loader.dll', + 'System.Runtime.Numerics.dll', + 'System.Runtime.Serialization.dll', + 'System.Runtime.Serialization.Formatters.dll', + 'System.Runtime.Serialization.Json.dll', + 'System.Runtime.Serialization.Primitives.dll', + 'System.Runtime.Serialization.Xml.dll', + 'System.Security.AccessControl.dll', + 'System.Security.Claims.dll', + 'System.Security.Cryptography.Algorithms.dll', + 'System.Security.Cryptography.Cng.dll', + 'System.Security.Cryptography.Csp.dll', + 'System.Security.Cryptography.Encoding.dll', + 'System.Security.Cryptography.OpenSsl.dll', + 'System.Security.Cryptography.Pkcs.dll', + 'System.Security.Cryptography.Primitives.dll', + 'System.Security.Cryptography.ProtectedData.dll', + 'System.Security.Cryptography.X509Certificates.dll', + 'System.Security.Cryptography.Xml.dll', + 'System.Security.dll', + 'System.Security.Permissions.dll', + 'System.Security.Principal.dll', + 'System.Security.Principal.Windows.dll', + 'System.Security.SecureString.dll', + 'System.ServiceModel.Syndication.dll', + 'System.ServiceModel.Web.dll', + 'System.ServiceProcess.dll', + 'System.ServiceProcess.ServiceController.dll', + 'System.Text.Encoding.CodePages.dll', + 'System.Text.Encoding.dll', + 'System.Text.Encoding.Extensions.dll', + 'System.Text.Encodings.Web.dll', + 'System.Text.Json.dll', + 'System.Text.RegularExpressions.dll', + 'System.Threading.AccessControl.dll', + 'System.Threading.Channels.dll', + 'System.Threading.dll', + 'System.Threading.Overlapped.dll', + 'System.Threading.Tasks.Dataflow.dll', + 'System.Threading.Tasks.dll', + 'System.Threading.Tasks.Extensions.dll', + 'System.Threading.Tasks.Parallel.dll', + 'System.Threading.Thread.dll', + 'System.Threading.ThreadPool.dll', + 'System.Threading.Timer.dll', + 'System.Transactions.dll', + 'System.Transactions.Local.dll', + 'System.Utf8String.Experimental.dll', + 'System.ValueTuple.dll', + 'System.Web.dll', + 'System.Web.HttpUtility.dll', + 'System.Windows.dll', + 'System.Windows.Extensions.dll', + 'System.Xml.dll', + 'System.Xml.Linq.dll', + 'System.Xml.ReaderWriter.dll', + 'System.Xml.Serialization.dll', + 'System.Xml.XDocument.dll', + 'System.Xml.XmlDocument.dll', + 'System.Xml.XmlSerializer.dll', + 'System.Xml.XPath.dll', + 'System.Xml.XPath.XDocument.dll', + 'WindowsBase.dll'] + +class CrossGenRunner: + def __init__(self, dotnet, crossgen_executable_filename): + self.dotnet = dotnet + self.crossgen_executable_filename = crossgen_executable_filename + self.platform_directory_sep = '\\' if sys.platform == 'win32' else '/' + + async def crossgen_il_file(self, il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch): + """ + Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /out {ni_filename} /in {il_filename}" + and returns returncode, stdour, stderr. + """ + args = self._build_args_crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch) + return await self._run_with_args(args) + + async def create_debugging_file(self, ni_filename, debugging_files_dirname, platform_assemblies_paths): + """ + Runs a subprocess "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /CreatePerfMap {debugging_files_dirname} /in {il_filename}" on Unix + or "{crossgen_executable_filename} /nologo /Platform_Assemblies_Paths /CreatePdb {debugging_files_dirname} /in {il_filename}" on Windows + and returns returncode, stdout, stderr. + """ + args = self._build_args_create_debugging_file(ni_filename, debugging_files_dirname, platform_assemblies_paths) + return await self._run_with_args(args) + + def _build_args_crossgen_il_file(self, il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch): + args = [] + args.append(self.dotnet) + args.append(self.crossgen_executable_filename) + args.append('-r') + args.append(platform_assemblies_paths + self.platform_directory_sep + '*.dll') + args.append('-O') + args.append('--out') + args.append(ni_filename) + args.append('--targetos ') + args.append(target_os) + args.append('--targetarch ') + args.append(target_arch) + args.append(il_filename) + return args + + def _build_args_create_debugging_file(self, ni_filename, debugging_files_dirname, platform_assemblies_paths): + args = [] + args.append(self.crossgen_executable_filename) + args.append('/nologo') + args.append('/Platform_Assemblies_Paths') + args.append(self.platform_assemblies_paths_sep.join(platform_assemblies_paths)) + args.append('/CreatePdb' if sys.platform == 'win32' else '/CreatePerfMap') + args.append(debugging_files_dirname) + args.append('/in') + args.append(ni_filename) + return args + + async def _run_with_args(self, args): + """ + Creates a subprocess running crossgen with specified set of arguments, + communicates with the owner process - waits for its termination and pulls + returncode, stdour, stderr. + """ + stdout = None + stderr = None + + proc = await asyncio.create_subprocess_shell(" ".join(args), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + stdout, stderr = await proc.communicate() + + return (proc.returncode, stdout.decode(), stderr.decode(), " ".join(args)) + + +def compute_file_hashsum(filename): + """ + Compute SHA256 file hashsum for {filename}. + """ + algo=hashlib.sha256() + maximum_block_size_in_bytes = 65536 + with open(filename, 'rb') as file: + while True: + block = file.read(maximum_block_size_in_bytes) + if block: + algo.update(block) + else: + break + return algo.hexdigest() + + +################################################################################ +# This describes collected during crossgen information. +################################################################################ +class CrossGenResult: + def __init__(self, assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, args): + self.assembly_name = assembly_name + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + self.output_file_hashsum = output_file_hashsum + self.output_file_size_in_bytes = output_file_size_in_bytes + self.output_file_type = output_file_type + self.args = args + +################################################################################ +# JSON Encoder for CrossGenResult objects. +################################################################################ +class CrossGenResultEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, CrossGenResult): + return { + 'AssemblyName': obj.assembly_name, + 'ReturnCode': obj.returncode, + 'StdOut': obj.stdout.splitlines(), + 'StdErr': obj.stderr.splitlines(), + 'OutputFileHash': obj.output_file_hashsum, + 'OutputFileSizeInBytes': obj.output_file_size_in_bytes, + 'OutputFileType': obj.output_file_type } + # Let the base class default method raise the TypeError + return json.JSONEncoder.default(self, obj) + +################################################################################ +# JSON Decoder for CrossGenResult objects. +################################################################################ +class CrossGenResultDecoder(json.JSONDecoder): + def __init__(self, *args, **kwargs): + json.JSONDecoder.__init__(self, object_hook=self._decode_object, *args, **kwargs) + def _decode_object(self, dict): + try: + assembly_name = dict['AssemblyName'] + returncode = dict['ReturnCode'] + stdout = dict['StdOut'] + stderr = dict['StdErr'] + output_file_hashsum = dict['OutputFileHash'] + output_file_size_in_bytes = dict['OutputFileSizeInBytes'] + output_file_type = dict['OutputFileType'] + return CrossGenResult(assembly_name, returncode, stdout, stderr, output_file_hashsum, output_file_size_in_bytes, output_file_type, "") + except KeyError: + return dict + + +################################################################################ +# Helper Functions +################################################################################ +def get_assembly_name(il_filename): + basename = os.path.basename(il_filename) + assembly_name, _ = os.path.splitext(basename) + return assembly_name + +class FileTypes: + NativeOrReadyToRunImage = 'NativeOrReadyToRunImage' + DebuggingFile = 'DebuggingFile' + +async def run_crossgen(dotnet, crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, target_os, target_arch): + runner = CrossGenRunner(dotnet, crossgen_executable_filename) + returncode, stdout, stderr, args = await runner.crossgen_il_file(il_filename, ni_filename, platform_assemblies_paths, target_os, target_arch) + ni_file_hashsum = compute_file_hashsum(ni_filename) if returncode == 0 else None + ni_file_size_in_bytes = os.path.getsize(ni_filename) if returncode == 0 else None + assembly_name = get_assembly_name(il_filename) + crossgen_assembly_result = CrossGenResult(assembly_name, returncode, stdout, stderr, ni_file_hashsum, ni_file_size_in_bytes, output_file_type=FileTypes.NativeOrReadyToRunImage, args=args) + + if returncode != 0: + return [crossgen_assembly_result] + +# Until crossgen2 can produce debugging file, don't do it + return [crossgen_assembly_result] + +# platform_assemblies_paths = platform_assemblies_paths + [os.path.dirname(ni_filename)] +# returncode, stdout, stderr = await runner.create_debugging_file(ni_filename, debugging_files_dirname, platform_assemblies_paths) + +# if returncode == 0: +# filenames = list(filter(lambda filename: not re.match("^{0}\.ni\.".format(assembly_name), filename, re.IGNORECASE) is None, os.listdir(debugging_files_dirname))) +# assert len(filenames) == 1 +# debugging_filename = os.path.join(debugging_files_dirname, filenames[0]) +# debugging_file_hashsum = compute_file_hashsum(debugging_filename) +# debugging_file_size_in_bytes = os.path.getsize(debugging_filename) +# else: +# debugging_file_hashsum = None +# debugging_file_size_in_bytes = None + +# create_debugging_file_result = CrossGenResult(assembly_name, returncode, stdout, stderr, debugging_file_hashsum, debugging_file_size_in_bytes, output_file_type=FileTypes.DebuggingFile) + +# return [crossgen_assembly_result, create_debugging_file_result] + + +def save_crossgen_result_to_json_file(crossgen_result, json_filename): + with open(json_filename, 'wt') as json_file: + json.dump(crossgen_result, json_file, cls=CrossGenResultEncoder, indent=2) + +def save_crossgen_results_to_json_files(crossgen_results, result_dirname): + for result in crossgen_results: + json_filename = os.path.join(result_dirname, "{0}-{1}.json".format(result.assembly_name, result.output_file_type)) + save_crossgen_result_to_json_file(result, json_filename) + +def create_output_folders(): + ni_files_dirname = tempfile.mkdtemp() + debugging_files_dirname = os.path.join(ni_files_dirname, "DebuggingFiles") + os.mkdir(debugging_files_dirname) + return ni_files_dirname, debugging_files_dirname + +async def crossgen_corelib(args): + il_corelib_filename = args.il_corelib_filename + assembly_name = os.path.basename(il_corelib_filename) + ni_corelib_dirname, debugging_files_dirname = create_output_folders() + ni_corelib_filename = os.path.join(ni_corelib_dirname, assembly_name) + platform_assemblies_paths = [os.path.dirname(il_corelib_filename)] + + # Validate the paths are correct. + if not os.path.exists(il_corelib_filename): + print("IL Corelib path does not exist.") + sys.exit(1) + + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + shutil.rmtree(ni_corelib_dirname, ignore_errors=True) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + +def add_ni_extension(filename): + filename,ext = os.path.splitext(filename) + return filename + '.ni' + ext + +def crossgen_framework(args): + ni_files_dirname, debugging_files_dirname = create_output_folders() + + async def run_crossgen_helper(print_prefix, assembly_name): + global g_frameworkcompile_failed + platform_assemblies_paths = args.core_root + print("{}{} {}".format(print_prefix, args.crossgen_executable_filename, assembly_name)) + + il_filename = os.path.join(args.core_root, assembly_name) + ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) + crossgen_results = await run_crossgen(args.dotnet, args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname, args.target_os, args.target_arch) + if crossgen_results[0].returncode != 0: + g_frameworkcompile_failed = True + print("{}{} {} return code={} args'{}'".format(print_prefix, args.crossgen_executable_filename, assembly_name, crossgen_results[0].returncode, crossgen_results[0].args)) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + + helper = AsyncSubprocessHelper(g_Framework_Assemblies, verbose=True) + helper.run_to_completion(run_crossgen_helper) + + shutil.rmtree(ni_files_dirname, ignore_errors=True) + if g_frameworkcompile_failed: + sys.exit(1) + +def load_crossgen_result_from_json_file(json_filename): + with open(json_filename, 'rt') as json_file: + return json.load(json_file, cls=CrossGenResultDecoder) + +def load_crossgen_results_from_dir(dirname, output_file_type): + crossgen_results = [] + for filename in glob.glob(os.path.join(dirname, '*.json')): + loaded_result = load_crossgen_result_from_json_file(filename) + if loaded_result.output_file_type == output_file_type: + crossgen_results.append(loaded_result) + return crossgen_results + +def dotnet_sdk_enumerate_assemblies(dotnet_sdk_dirname): + for dirpath, _, filenames in os.walk(dotnet_sdk_dirname): + dirname = os.path.dirname(dirpath) + if dirname.endswith('Microsoft.NETCore.App') or dirname.endswith('Microsoft.AspNetCore.App') or dirname.endswith('Microsoft.AspNetCore.All'): + filenames = filter(lambda filename: not re.match(r'^(Microsoft|System)\..*dll$', filename) is None, filenames) + filenames = filter(lambda filename: filename != 'System.Private.CoreLib.dll', filenames) + yield (dirpath, filenames) + +async def crossgen_dotnet_sdk(args): + dotnet_sdk_dirname = tempfile.mkdtemp() + with tarfile.open(args.dotnet_sdk_filename) as dotnet_sdk_tarfile: + dotnet_sdk_tarfile.extractall(dotnet_sdk_dirname) + + il_corelib_filename = args.il_corelib_filename + ni_files_dirname, debugging_files_dirname = create_output_folders() + ni_corelib_filename = os.path.join(ni_files_dirname, os.path.basename(il_corelib_filename)) + platform_assemblies_paths = [os.path.dirname(il_corelib_filename)] + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_corelib_filename, ni_corelib_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + + platform_assemblies_paths = [ni_files_dirname] + + for il_files_dirname, _ in dotnet_sdk_enumerate_assemblies(dotnet_sdk_dirname): + platform_assemblies_paths.append(il_files_dirname) + + for il_files_dirname, assembly_names in dotnet_sdk_enumerate_assemblies(dotnet_sdk_dirname): + for assembly_name in assembly_names: + il_filename = os.path.join(il_files_dirname, assembly_name) + ni_filename = os.path.join(ni_files_dirname, add_ni_extension(assembly_name)) + crossgen_results = await run_crossgen(args.crossgen_executable_filename, il_filename, ni_filename, platform_assemblies_paths, debugging_files_dirname) + save_crossgen_results_to_json_files(crossgen_results, args.result_dirname) + shutil.rmtree(ni_files_dirname, ignore_errors=True) + +def print_omitted_assemblies_message(omitted_assemblies, dirname): + print('The information for the following assemblies was omitted from "{0}" directory:'.format(dirname)) + for assembly_name in sorted(omitted_assemblies): + print(' - ' + assembly_name) + +def print_compare_result_message_helper(message_header, base_value, diff_value, base_dirname, diff_dirname): + assert base_value != diff_value + print(message_header) + print(' - "{0}" has "{1}"'.format(base_dirname, base_value)) + print(' - "{0}" has "{1}"'.format(diff_dirname, diff_value)) + +def compare_and_print_message(base_result, diff_result, base_dirname, diff_dirname): + base_diff_are_equal = True + + assert base_result.assembly_name == diff_result.assembly_name + assert base_result.output_file_type == diff_result.output_file_type + + if base_result.returncode != diff_result.returncode: + base_diff_are_equal = False + print_compare_result_message_helper('Return code mismatch for "{0}" assembly for files of type "{1}":'.format(base_result.assembly_name, base_result.output_file_type), base_result.returncode, diff_result.returncode, base_dirname, diff_dirname) + elif base_result.returncode == 0 and diff_result.returncode == 0: + assert not base_result.output_file_hashsum is None + assert not base_result.output_file_size_in_bytes is None + + assert not diff_result.output_file_hashsum is None + assert not diff_result.output_file_size_in_bytes is None + + if base_result.output_file_hashsum != diff_result.output_file_hashsum: + base_diff_are_equal = False + print_compare_result_message_helper('File hash sum mismatch for "{0}" assembly for files of type "{1}":'.format(base_result.assembly_name, base_result.output_file_type), base_result.output_file_hashsum, diff_result.output_file_hashsum, base_dirname, diff_dirname) + + if base_result.output_file_size_in_bytes != diff_result.output_file_size_in_bytes: + base_diff_are_equal = False + print_compare_result_message_helper('File size mismatch for "{0}" assembly for files of type "{1}":'.format(base_result.assembly_name, base_result.output_file_type), base_result.output_file_size_in_bytes, diff_result.output_file_size_in_bytes, base_dirname, diff_dirname) + + return base_diff_are_equal + +def compare_results(args): + """ + Checks whether {base} and {diff} crossgens are "equal": + 1. their return codes are the same; + 2. and if they both succeeded in step 1, their outputs (native images and debugging files (i.e. pdb or perfmap files)) are the same. + """ + base_diff_are_equal = True + did_compare = False + + for output_file_type in [FileTypes.NativeOrReadyToRunImage, FileTypes.DebuggingFile]: + print('Comparing crossgen results in "{0}" and "{1}" directories for files of type "{2}":'.format(args.base_dirname, args.diff_dirname, output_file_type)) + + base_results = load_crossgen_results_from_dir(args.base_dirname, output_file_type) + diff_results = load_crossgen_results_from_dir(args.diff_dirname, output_file_type) + + base_assemblies = { r.assembly_name for r in base_results } + diff_assemblies = { r.assembly_name for r in diff_results } + both_assemblies = base_assemblies & diff_assemblies + + num_omitted_results = 0 + + omitted_from_base_dir = diff_assemblies - base_assemblies + omitted_from_diff_dir = base_assemblies - diff_assemblies + + if len(omitted_from_base_dir) != 0: + num_omitted_results += len(omitted_from_base_dir) + base_diff_are_equal = False + print_omitted_assemblies_message(omitted_from_base_dir, args.base_dirname) + + if len(omitted_from_diff_dir) != 0: + num_omitted_results += len(omitted_from_diff_dir) + base_diff_are_equal = False + print_omitted_assemblies_message(omitted_from_diff_dir, args.diff_dirname) + + base_results_by_name = dict((r.assembly_name, r) for r in base_results) + diff_results_by_name = dict((r.assembly_name, r) for r in diff_results) + + num_mismatched_results = 0 + + for assembly_name in sorted(both_assemblies): + base_result = base_results_by_name[assembly_name] + diff_result = diff_results_by_name[assembly_name] + did_compare = True + if not compare_and_print_message(base_result, diff_result, args.base_dirname, args.diff_dirname): + base_diff_are_equal = False + num_mismatched_results += 1 + + print("Number of omitted results: {0}".format(num_omitted_results)) + print("Number of mismatched results: {0}".format(num_mismatched_results)) + print("Total number of files compared: {0}".format(len(both_assemblies))) + + if not did_compare: + sys.exit(1) + + sys.exit(0 if base_diff_are_equal else 1) + +################################################################################ +# __main__ +################################################################################ + +if __name__ == '__main__': + start = datetime.datetime.now() + + parser = build_argument_parser() + args = parser.parse_args() + func = args.func(args) + + end = datetime.datetime.now() + elapsed = end - start + + print("Elapsed time: {}".format(elapsed.total_seconds()))