azure-pipelines.yml

variables:
  buildNumber: $[ counter('constant', 13000) ] # Start higher than our AppVeyor versions. Every build (pr or branch) will increment.

name: 3.0.$(buildNumber)
  
pr:
  branches:
    include:
    - dev
    - release/3.0*
    - release/ExtensionsMetadataGenerator/*

trigger:
  branches:
    include:
    - dev
    - release/3.0*
    - release/ExtensionsMetadataGenerator/*

jobs:
- job: InitializePipeline
  pool:
    vmImage: 'windows-2019'
  steps:
  - task: AzureKeyVault@1
    inputs:
      # Note: This is actually a Service Connection in DevOps, not an Azure subscription name
      azureSubscription: 'Azure-Functions-Host-CI'
      keyVaultName: 'azure-functions-host-ci'
      secretsFilter: '*'
  - task: PowerShell@2
    displayName: 'Initialize'
    name: Initialize
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\initialize-pipeline.ps1'
      arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'''

- job: BuildArtifacts
  dependsOn: InitializePipeline
  condition: and(succeeded(), or(ne(variables['Build.Reason'], 'PullRequest'), eq(dependencies.InitializePipeline.outputs['Initialize.BuildArtifacts'], true)))
  variables:
    ${{ if or( eq( variables['Build.Reason'], 'PullRequest' ), and( not( contains( variables['Build.SourceBranch'], 'release/3.0' ) ), not( contains( variables['Build.SourceBranch'], 'release/ExtensionsMetadataGenerator/' ) ) ) ) }}:
      suffixTemp: ci
      packSuffixSwitchTemp: --version-suffix ci
      emgSuffixSwitchTemp: --version-suffix ci$(buildNumber)
      artifactSuffix: -ci
    suffix: $[variables.suffixTemp] # this resolves to an empty string if it is missing
    packSuffixSwitch: $[variables.packSuffixSwitchTemp]
    emgSuffixSwitch: $[variables.emgSuffixSwitchTemp]
  pool:
    vmImage: 'windows-2019' 
  steps:
  - task: UseDotNet@2
    inputs:
      packageType: 'sdk'
      version: '3.1.x'
      performMultiLevelLookup: true
  - task: PowerShell@2
    displayName: "Build artifacts"
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\build-extensions.ps1'
      arguments: '-buildNumber "$(buildNumber)" -suffix "$(suffix)" -commitHash "$(Build.SourceVersion)"'
  - task: PowerShell@2
    condition: eq(variables['RUNBUILDFORINTEGRATIONTESTS'], 'True')
    displayName: "Update host references"
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\update-hostreferences.ps1'
  - task: CopyFiles@2
    inputs:
      SourceFolder: '$(Build.Repository.LocalPath)\buildoutput'
      Contents: '**\*.zip'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'
  - task: DotNetCoreCLI@2
    displayName: 'Build host packages'
    inputs:
      command: 'custom'
      custom: 'pack'
      arguments: -o packages -p:BuildNumber=$(buildNumber) -p:CommitHash=$(Build.SourceVersion) -c Release $(packSuffixSwitch)
      projects: |
        **\WebJobs.Script.csproj
        **\WebJobs.Script.WebHost.csproj
        **\WebJobs.Script.Grpc.csproj
  
  - pwsh: |
      foreach ($baseName in @("WebJobs.Script", "WebJobs.Script.WebHost", "WebJobs.Script.Grpc"))
      {    
        $packageName = "Microsoft.Azure." + $baseName + "*.nupkg"
        $sourcePath = "$(Build.Repository.LocalPath)/packages/$packageName"
        if (-not (test-path $sourcePath))    
        {
          throw "Unable to find '$packageName' at './package'"
        }
        Copy-Item -Path $sourcePath -Destination $(Build.ArtifactStagingDirectory) -ErrorAction Stop -Verbose -Force}
    condition: eq(variables['RUNBUILDFORINTEGRATIONTESTS'], 'True')
    displayName: 'Copy package to ArtifactStagingDirectory'

  - task: NuGetCommand@2
    condition: eq(variables['RUNBUILDFORINTEGRATIONTESTS'], 'True')
    inputs:
      command: 'push'
      packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg'
      nuGetFeedType: 'internal'
      publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
      allowPackageConflicts: true
 
  - task: DotNetCoreCLI@2
    displayName: 'Build performance package'
    inputs:
      command: 'custom'
      custom: 'pack'
      arguments: '-o WebJobs.Script.Performance.App'
      projects: |
        **\WebJobs.Script.Performance.App.csproj
  - task: DotNetCoreCLI@2
    displayName: 'Build Abstractions and ExtensionsMetadataGenerator'
    inputs:
      command: 'build'
      arguments: '-c Release -p:CommitHash=$(Build.SourceVersion)'
      projects: |
        **\ExtensionsMetadataGenerator.csproj
        **\WebJobs.Script.Abstractions.csproj
  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
    displayName: 'ESRP CodeSigning: Strong Name and Authenticode'
    inputs:
      ConnectedServiceName: 'ESRP Service'
      FolderPath: 'src\WebJobs.Script.Abstractions\bin\Release'
      Pattern: Microsoft.Azure.WebJobs.Script.Abstractions*.dll
      signConfigType: inlineSignParams
      inlineOperation: |
        [
          {
            "KeyCode": "CP-230012",
            "OperationCode": "SigntoolSign",
            "Parameters": {
              "OpusName": "Microsoft",
              "OpusInfo": "http://www.microsoft.com",
              "FileDigest": "/fd \"SHA256\"",
              "PageHash": "/NPH",
              "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
            },
            "ToolName": "sign",
            "ToolVersion": "1.0"
          },
          {
            "KeyCode": "CP-230012",
            "OperationCode": "SigntoolVerify",
            "Parameters": {},
            "ToolName": "sign",
            "ToolVersion": "1.0"
          }
        ]
  - task: DeleteFiles@1
    displayName: 'Delete CodeSignSummary files'
    inputs:
      contents: '**\CodeSignSummary-*.md'
  - task: DotNetCoreCLI@2
    displayName: 'Pack WebJobs.Script.Abstractions package'
    inputs:
      command: 'custom'
      custom: 'pack'
      arguments: '--no-build -c Release -o packages'
      projects: |
        **\WebJobs.Script.Abstractions.csproj
  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
    displayName: 'ESRP CodeSigning: Nupkg'
    inputs:
      ConnectedServiceName: 'ESRP Service'
      FolderPath: 'packages'
      Pattern: 'Microsoft.Azure.WebJobs.Script.Abstractions*.nupkg'
      signConfigType: inlineSignParams
      inlineOperation: |
        [
            {
              "KeyCode": "CP-401405",
              "OperationCode": "NuGetSign",
              "Parameters": {},
              "ToolName": "sign",
              "ToolVersion": "1.0"
            },
            {
              "KeyCode": "CP-401405",
              "OperationCode": "NuGetVerify",
              "Parameters": {},
              "ToolName": "sign",
              "ToolVersion": "1.0"
            }
        ]
  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
    displayName: 'ESRP CodeSigning: Strong Name and Authenticode'
    inputs:
      ConnectedServiceName: 'ESRP Service'
      FolderPath: 'tools\ExtensionsMetadataGenerator\src\ExtensionsMetadataGenerator\bin\Release'
      Pattern: Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator*.dll
      signConfigType: inlineSignParams
      inlineOperation: |
        [
          {
              "KeyCode" : "CP-233863-SN",
              "OperationCode" : "StrongNameSign",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
          },
          {
              "KeyCode" : "CP-233863-SN",
              "OperationCode" : "StrongNameVerify",
              "Parameters" : {},
              "ToolName" : "sign",
              "ToolVersion" : "1.0"
          },
          {
            "KeyCode": "CP-230012",
            "OperationCode": "SigntoolSign",
            "Parameters": {
              "OpusName": "Microsoft",
              "OpusInfo": "http://www.microsoft.com",
              "FileDigest": "/fd \"SHA256\"",
              "PageHash": "/NPH",
              "TimeStamp": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256"
            },
            "ToolName": "sign",
            "ToolVersion": "1.0"
          },
          {
            "KeyCode": "CP-230012",
            "OperationCode": "SigntoolVerify",
            "Parameters": {},
            "ToolName": "sign",
            "ToolVersion": "1.0"
          }
        ]
    condition: and(succeeded(), startsWith(variables['SignArtifacts'], 'true'))  
  - task: DeleteFiles@1
    displayName: 'Delete CodeSignSummary files'
    inputs:
      contents: '**\CodeSignSummary-*.md'
  - task: DotNetCoreCLI@2
    displayName: 'Pack ExtensionsMetadataGenerator package'
    inputs:
      command: 'custom'
      custom: 'pack'
      arguments: '--no-build -c Release -o packages $(emgSuffixSwitch)'
      projects: |
        **\ExtensionsMetadataGenerator.csproj
        steps:
  - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
    displayName: 'ESRP CodeSigning: Nupkg'
    inputs:
      ConnectedServiceName: 'ESRP Service'
      FolderPath: 'packages'
      Pattern: 'Microsoft.Azure.WebJobs.Script.ExtensionsMetadataGenerator*.nupkg'
      signConfigType: inlineSignParams
      inlineOperation: |
       [
           {
             "KeyCode": "CP-401405",
             "OperationCode": "NuGetSign",
             "Parameters": {},
             "ToolName": "sign",
             "ToolVersion": "1.0"
           },
           {
             "KeyCode": "CP-401405",
             "OperationCode": "NuGetVerify",
             "Parameters": {},
             "ToolName": "sign",
             "ToolVersion": "1.0"
           }
       ]
    condition: and(succeeded(), startsWith(variables['SignArtifacts'], 'true'))
  - task: DeleteFiles@1
    displayName: 'Delete CodeSignSummary files'
    inputs:
      contents: '**\CodeSignSummary-*.md'
  - publish: $(Build.ArtifactStagingDirectory)\V2SiteExtension
    artifact: V2SiteExtension
  - publish: $(Build.ArtifactStagingDirectory)\SiteExtension
    artifact: SiteExtension
  - publish: $(Build.ArtifactStagingDirectory)\PrivateSiteExtension
    artifact: PrivateSiteExtension
  - publish: $(Build.ArtifactStagingDirectory)\Symbols
    artifact: Symbols
  - publish: $(Build.Repository.LocalPath)\packages
    artifact: NugetPackages
  - publish: $(Build.Repository.LocalPath)\WebJobs.Script.Performance.App
    artifact: Performance

- job: RunUnitTests
  pool:
    vmImage: 'windows-2019'
  steps: 
  - task: UseDotNet@2
    inputs:
      packageType: 'sdk'
      version: '3.1.x'
      performMultiLevelLookup: true
  - task: DotNetCoreCLI@2
    displayName: 'Unit Tests'
    inputs:
      command: 'test'
      testRunTitle: 'Unit Tests'
      arguments: '-v n'
      projects: |
        **\ExtensionsMetadataGeneratorTests.csproj
        **\WebJobs.Script.Scaling.Tests.csproj
        **\WebJobs.Script.Tests.csproj

- job: RunNonE2EIntegrationTests
  pool:
    vmImage: 'windows-2019'
  steps: 
  - task: UseDotNet@2
    inputs:
      packageType: 'sdk'
      version: '3.1.x'
      performMultiLevelLookup: true
  - task: UseNode@1
    inputs:      
      version: '10.x'
  - task: PowerShell@2
    displayName: 'Install Az.Storage Powershell module'
    inputs:
      targetType: 'inline'
      script: 'Install-Module -Name Az.Storage -RequiredVersion 1.11.0 -Scope CurrentUser -Force -AllowClobber'
  - task: AzureKeyVault@1
    inputs:
      # Note: This is actually a Service Connection in DevOps, not an Azure subscription name
      azureSubscription: 'Azure-Functions-Host-CI'
      keyVaultName: 'azure-functions-host-ci'
      secretsFilter: '*'
  - task: PowerShell@2
    displayName: 'Checkout secrets'
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\checkout-secrets.ps1'
      arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'''
  - task: AzureKeyVault@1
    inputs:
      # Note: This is actually a Service Connection in DevOps, not an Azure subscription name
      azureSubscription: 'Azure-Functions-Host-CI'
      keyVaultName: azure-functions-host-$(LeaseBlob)
      secretsFilter: '*'
  - task: DotNetCoreCLI@2
    displayName: 'Non-E2E integration tests'
    inputs:
      command: 'test'
      testRunTitle: 'Non-E2E integration tests'
      arguments: '--filter "Category!=E2E"'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
    env:
      AzureWebJobsStorage: $(Storage)
      AzureWebJobsSecondaryStorage: $(SecondaryStorage)
      ConnectionStrings__CosmosDB: $(CosmosDB)
      AzureWebJobsEventHubSender: $(EventHub)
      AzureWebJobsEventHubReceiver: $(EventHub)
      AzureWebJobsSecretStorageKeyVaultConnectionString: $(KeyVaultConnectionString)
      AzureWebJobsSecretStorageKeyVaultName: $(KeyVaultName)
  - task: PowerShell@2
    condition: always()
    displayName: 'Checkin secrets'
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\checkin-secrets.ps1'
      arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'' -leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'

- job: RunIntegrationTests
  pool:
    vmImage: 'windows-2019'
  steps: 
  - task: UseDotNet@2
    inputs:
      packageType: 'sdk'
      version: '3.1.x'
      performMultiLevelLookup: true
  - task: UseNode@1
    inputs:
      version: '10.x'
  - task: UsePythonVersion@0
    inputs:
      versionSpec: '3.7.x'
      addToPath: true
  - task: PowerShell@2
    displayName: 'Install Az.Storage Powershell module'
    inputs:
      targetType: 'inline'
      script: 'Install-Module -Name Az.Storage -RequiredVersion 1.11.0 -Scope CurrentUser -Force -AllowClobber'
  - task: AzureKeyVault@1
    inputs:
      # Note: This is actually a Service Connection in DevOps, not an Azure subscription name
      azureSubscription: 'Azure-Functions-Host-CI'
      keyVaultName: 'azure-functions-host-ci'
      secretsFilter: '*'
  - task: PowerShell@2
    displayName: 'Checkout secrets'
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\checkout-secrets.ps1'
      arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'''
  - task: AzureKeyVault@1
    inputs:
      # Note: This is actually a Service Connection in DevOps, not an Azure subscription name
      azureSubscription: 'Azure-Functions-Host-CI'
      keyVaultName: azure-functions-host-$(LeaseBlob)
      secretsFilter: '*'
  - task: PowerShell@2
    displayName: 'Set environment variables'
    inputs:
      targetType: 'inline'
      script: |
        Write-Host "##vso[task.setvariable variable=AzureWebJobsStorage]$env:AzureWebJobsStorageSecretMap"
        Write-Host "##vso[task.setvariable variable=AzureWebJobsSeconaryStorage]$env:AzureWebJobsSecondaryStorageSecretMap"
        Write-Host "##vso[task.setvariable variable=ConnectionStrings__CosmosDB]$env:CosmosDbSecretMap"
        Write-Host "##vso[task.setvariable variable=AzureWebJobsEventHubSender]$env:AzureWebJobsEventHubSenderSecretMap"
        Write-Host "##vso[task.setvariable variable=AzureWebJobsEventHubReceiver]$env:AzureWebJobsEventHubReceiverSecretMap"
    env:
      AzureWebJobsStorageSecretMap: $(Storage)
      AzureWebJobsSecondaryStorageSecretMap: $(SecondaryStorage)
      CosmosDbSecretMap: $(CosmosDb)
      AzureWebJobsEventHubSenderSecretMap: $(EventHub)
      AzureWebJobsEventHubReceiverSecretMap: $(EventHub)
  - task: DotNetCoreCLI@2
    displayName: "C# end to end tests"
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "C# end to end tests"
      arguments: '--filter "Group=CSharpEndToEndTests"'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Node end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Node end to end tests"
      arguments: '--filter "Group=NodeEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Direct load end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Direct load end to end tests"
      arguments: '--filter "Group=DirectLoadEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "F# end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "F# end to end tests"
      arguments: '--filter "Group=FSharpEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Language worker end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Language worker end to end tests"
      arguments: '--filter "Group=LanguageWorkerSelectionEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Node script host end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Node script host end to end tests"
      arguments: '--filter "Group=NodeScriptHostTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Raw assembly end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Raw assembly end to end tests"
      arguments: '--filter "Group=RawAssemblyEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Samples end to end tests"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Samples end to end tests"
      arguments: '--filter "Group=SamplesEndToEndTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Standby mode end to end tests Windows"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Standby mode end to end tests Windows"
      arguments: '--filter "Group=StandbyModeEndToEndTests_Windows" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Standby mode end to end tests Linux"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Standby mode end to end tests Linux"
      arguments: '--filter "Group=StandbyModeEndToEndTests_Linux" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: DotNetCoreCLI@2
    displayName: "Linux container end to end tests Windows"    
    condition: succeededOrFailed()
    inputs:
      command: 'test'
      testRunTitle: "Linux container end to end tests Windows"
      arguments: '--filter "Group=ContainerInstanceTests" --no-build'
      projects: |
        **\WebJobs.Script.Tests.Integration.csproj
  - task: PowerShell@2
    condition: always()
    displayName: 'Checkin secrets'
    inputs:
      filePath: '$(Build.Repository.LocalPath)\build\checkin-secrets.ps1'
      arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'' -leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'