Add azure-pipelines build and test definitions (dotnet/coreclr#20840)

This adds an azure pipeline definition with a matrix of product and test builds, using helix to run tests. The intention is that this definition will eventually be used for both our official build and CI testing.

There is one build job for each OS/platform/arch, and one test job for each OS/platform/arch/priority/R2Rflag. The test job builds tests and then submits them to helix, passing along a number of test run modes. One helix test job will be created for each OS/platform/arch/priority/R2Rflag/helixtargetqueue/testscenario.

There is a lot of work left to be done to get this up to parity with our official builds and CI, which I've tried to call out in comments.

Commit migrated from dotnet/coreclr@8bc5616

eng/build-job.yml

@@ -0,0 +1,108 @@
+parameters:
+  buildConfig: ''
+  archType: ''
+  osGroup: ''
+
+### Product build
+jobs:
+- template: xplat-job.yml
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: ${{ parameters.archType }}
+    osGroup: ${{ parameters.osGroup }}
+
+    # Compute job name from template parameters
+    name: ${{ format('build_{0}_{1}_{2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+    displayName: ${{ format('Build {0} {1} {2}', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+
+    steps:
+
+    # Install native dependencies
+    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+      - script: sh eng/install-native-dependencies.sh $(osGroup)
+        displayName: Install native dependencies
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      # Necessary to install python
+      - script: eng\common\init-tools-native.cmd -InstallDirectory $(Build.SourcesDirectory)\native-tools -Force
+        displayName: Install native dependencies
+
+
+    # Run init-tools (pre-arcade dependency bootstrapping)
+    # TODO: replace this with an arcade equivalent
+    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+      - script: ./init-tools.sh
+        displayName: Init tools
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      - script: .\init-tools.cmd
+        displayName: Init tools
+
+
+    # Sync
+    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+      - script: ./Tools/dotnetcli/dotnet msbuild build.proj /p:RestoreDuringBuild=true /t:Sync
+        displayName: Sync
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      - script: .\Tools\dotnetcli\dotnet.exe msbuild build.proj /p:RestoreDuringBuild=true /t:Sync
+        displayName: Sync
+
+
+    # Build
+    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+      - script: ./build.sh $(buildConfig) $(archType) -skipnuget -skiprestore
+        displayName: Build product
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      # TODO: IBCOptimize? EnforcePGO? pass an OfficialBuildId? SignType? file logging parameters?
+      - script: set __TestIntermediateDir=int&&build.cmd $(buildConfig) $(archType) -skiptests -skipbuildpackages -skiprestore
+        displayName: Build product
+
+
+    # Upload build as pipeline artifact
+    - ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+      - task: PublishPipelineArtifact@0
+        displayName: Save product build as pipeline artifact
+        inputs:
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          targetPath: $(Build.SourcesDirectory)/bin/Product/$(osGroup).$(archType).$(buildConfigUpper)
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      - task: PublishPipelineArtifact@0
+        displayName: Save product build as pipeline artifact
+        inputs:
+          artifactName: ${{ format('{0}_{1}_{2}_build', parameters.osGroup, parameters.archType, parameters.buildConfig) }}
+          targetPath: $(Build.SourcesDirectory)\bin\Product\Windows_NT.$(archType).$(buildConfigUpper)
+
+
+    # TODO: Sign
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - script: echo Sign!
+        displayName: Sign Binaries (empty for now)
+
+
+    # Get key vault secrets for publishing
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: AzureKeyVault@1
+        inputs:
+          azureSubscription: 'DotNet-Engineering-Services_KeyVault'
+          KeyVaultName: EngKeyVault
+          SecretsFilter: 'dotnetfeed-storage-access-key-1,microsoft-symbol-server-pat,symweb-symbol-server-pat'
+
+
+    # TODO: Build packages and publish official build
+    #- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    #    ${{ if or(eq(parameters.osGroup, 'Linux'), eq(parameters.osGroup, 'OSX')) }}:
+    #      # TODO: ensure that NUGET_PACKAGES path is correctly set
+    #      - script: .dotnet/dotnet msbuild src/restorearcadepublishtasks.proj /t:Restore
+    #        displayName: Restore tasks used to publish the build
+    #      - script: .dotnet/dotnet msbuild src/publishwitharcade.proj /t:PublishPackages /p:AzureFeedUrl=$(AzureFeedUrl) /p:AccountKey=$(AccountKey)
+    #        displayName: Publish official build
+    #    ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+    #      # TODO: pass publish feed url and access token in from the internal pipeline
+    #      - script: .dotnet\dotnet.exe msbuild src\restorearcadepublishtasks.proj /t:Restore
+    #        displayName: Restore tasks used to publish the build
+    #      - script: .dotnet\dotnet.exe msbuild src\publishwitharcade.proj /t:PublishPackages /p:AzureFeedUrl=$(AzureFeedUrl) /p:AccountKey=$(AccountKey)
+
+    # Kill tasks that hold onto files on Windows. Otherwise git clean
+    # may fail for later jobs on the same agent.
+    - ${{ if eq(parameters.osGroup, 'Windows_NT') }}:
+      - script: eng/kill_tasks.cmd
+        displayName: Kill tasks that hold on to files
+        condition: always()

eng/install-native-dependencies.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env sh
+
+if [ "$1" = "Linux" ]; then
+    sudo apt update
+    if [ "$?" != "0" ]; then
+       exit 1;
+    fi
+    sudo apt install cmake llvm-3.9 clang-3.9 lldb-3.9 liblldb-3.9-dev libunwind8 libunwind8-dev gettext libicu-dev liblttng-ust-dev libcurl4-openssl-dev libssl-dev libkrb5-dev libnuma-dev
+    if [ "$?" != "0"]; then
+        exit 1;
+    fi
+elif [ "$1" = "OSX" ]; then
+    brew install icu4c openssl
+    if [ "$?" != "0" ]; then
+        exit 1;
+    fi
+    brew link --force icu4c
+    if [ "$?" != "0"]; then
+        exit 1;
+    fi
+else
+    echo "Must pass \"Linux\" or \"OSX\" as first argument."
+    exit 1
+fi
+

eng/kill_tasks.cmd

@@ -0,0 +1,10 @@
+@if not defined _echo @echo off
+setlocal EnableDelayedExpansion
+
+:: Check if VBCSCompiler.exe is running
+tasklist /fi "imagename eq VBCSCompiler.exe" |find ":" > nul
+:: Compiler is running if errorlevel == 1
+if errorlevel 1 (
+	echo Stop VBCSCompiler.exe execution.
+	for /f "tokens=2 delims=," %%F in ('tasklist /nh /fi "imagename eq VBCSCompiler.exe" /fo csv') do taskkill /f /PID %%~F
+)

eng/pipelines/coreclr/azure-pipelines.yml

@@ -0,0 +1,178 @@
+resources:
+  repositories:
+  # shared library repository
+  - repository: arcade
+    type: github
+    endpoint: DotNet-Bot GitHub Connection
+    name: dotnet/arcade
+    ref: refs/heads/master
+
+variables:
+  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
+
+
+jobs:
+
+##   The following is the matrix of test runs that we have. This is
+##   duplicated for each os/arch combination in platform-matrix.yml.
+
+##
+##   Product build       Test build              Test run
+##   (Azure DevOps)      (Azure DevOps)          (helix)
+##
+##   ###########################################################################################
+##
+##   Debug
+##
+##   Checked ----------> Pri0 -----------------> plain runtests
+##           |
+##           \---------> Pri1 -----------------> plain runtests
+##           |                \----------------> jitstress
+##           |                \----------------> gcstress
+##           |                \----------------> maybe more (dynamically selected runtest modes)
+##           |
+##           \---------> Pri1 crossgen --------> plain runtests
+##                                     \-------> jitstress
+##                                     \-------> gcstress
+##                                     \-------> maybe more (dynamically selected runtest modes)
+##
+##   Release ----------> Pri1 -----------------> plain runtests
+##           |
+##           \---------> Pri1 crossgen --------> plain runtests
+##
+##
+
+## Each build or test job is defined in Azure DevOps, and will show
+## up in the UI in the order in which they are defined here. The
+## build and test build job matrix is defined statically, but
+## queue-time inputs can be used to control whether a job executes
+## (used to select which jobs run in ci vs for official builds), or
+## to select test modes. This should eventually be used to enable
+## requesting specific test runs from pull requests.
+
+
+##
+## Templates used to define jobs:
+## Please update this if the factoring changes.
+##
+## This file defines the set of jobs in a platform-agnostic manner,
+## using the platform-matrix.yml template. This will create one job
+## for each platform from the passed-in jobTemplate (either a build
+## job or a test job). The build-job.yml and test-job.yml templates
+## use xplat-job.yml to handle some of the common logic for
+## abstracting over platforms. Finally, xplat-job.yml uses the arcade
+## base.yml job template, which sets up telemetry and signing support.
+
+## azure-pipelines.yml -> platform-matrix.yml -------> build-job.yml -------> xplat-job.yml -> base.yml
+##                                            |  (passed-in jobTemplate)  |                    (arcade)
+##                                            \------> test-job.yml ------/
+
+
+
+#
+# Debug build
+#
+
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: build-job.yml
+    buildConfig: debug
+
+#
+# Checked build
+#
+
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: build-job.yml
+    buildConfig: checked
+
+#
+# Release build
+#
+
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: build-job.yml
+    buildConfig: release
+
+#
+# Checked test builds
+#
+
+# Pri0
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: checked
+    jobParameters:
+      priority: 0
+
+# Pri1
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: checked
+    jobParameters:
+      priority: 1
+      scenarios: 'normal;jitstress2'
+
+# Pri1 crossgen
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: checked
+    jobParameters:
+      priority: 1
+      crossgen: true
+      scenarios: 'normal;jitstress2'
+
+#
+# Release test builds
+#
+
+# Pri1
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: release
+    jobParameters:
+      priority: 1
+
+# Pri1 crossgen
+- template: eng/platform-matrix.yml
+  parameters:
+    jobTemplate: test-job.yml
+    buildConfig: release
+    jobParameters:
+      priority: 1
+      crossgen: true
+
+
+# Publish build information to Build Assets Registry
+
+# This job gathers build assets from the pipeline (from each official
+# product build job), and publishes them to the build assets
+# registry. Its dependencies should be updated to include all of the
+# official builds if we add more platform/arch combinations.
+
+# TODO: Enable publish to BAR
+#- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+#  - template: /eng/common/templates/phases/publish-build-assets.yml@arcade
+#- phase: publish_bar
+#  displayName: publish to BAR (empty currently)
+#  queue:
+#    name: Hosted VS2017
+#  dependsOn:
+#  - build_Linux_x64_release
+#  - build_OSX_x64_release
+#  - build_Windows_NT_x64_release
+#  - build_Windows_NT_x86_release
+#  - build_Windows_NT_arm_release
+#  - build_Windows_NT_arm64_release
+  # TODO: enable these builds
+  #  - build_rhel_x64_release
+  #  - build_alpine_x64_release
+  #  - build_crossbuild_arm_release
+  #  - build_crossbuild_arm64_release
+

eng/platform-matrix.yml

@@ -0,0 +1,61 @@
+parameters:
+  jobTemplate: ''
+  buildConfig: ''
+  jobParameters: {}
+
+jobs:
+
+# Linux x64
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Linux
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# macOS x64
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: OSX
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# Windows x64/x86/arm/arm64
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x64
+    osGroup: Windows_NT
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: x86
+    osGroup: Windows_NT
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm
+    osGroup: Windows_NT
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+- template: ${{ parameters.jobTemplate }}
+  parameters:
+    buildConfig: ${{ parameters.buildConfig }}
+    archType: arm64
+    osGroup: Windows_NT
+    ${{ insert }}: ${{ parameters.jobParameters }}
+
+# TODO for official build:
+# RedHat x64
+# Linux crossbuild arm
+# Linux crossbuild arm64
+# Linux musl x64
+