Add internal build support (dotnet/core-setup#8279)

Add NuGetAuthenticate task.

Create proxy build scripts that include pre-build auth plugin install.

Add workaround to avoid running test restore against authenticated feed.

Pass the NuGet auth env vars set by the AzDO build step into the Docker container.

Add configurability for internal publish.

Commit migrated from dotnet/core-setup@a896b3e

eng/install-nuget-credprovider-then-build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+# Installs the NuGet Credential Provider, then calls ../build.sh with all arguments. This creates a
+# build context that can restore from authenticated sources. This script is intended for use by the
+# official Microsoft build inside a Docker container.
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/install-nuget-credprovider.sh"
+
+"$scriptroot/../build.sh" "$@"

eng/install-nuget-credprovider-then-msbuild.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -e
+
+# Installs the NuGet Credential Provider, then calls common/msbuild.sh with all arguments. This
+# creates a build context that can restore from authenticated sources. This script is intended for
+# use by the official Microsoft build inside a Docker container.
+
+source="${BASH_SOURCE[0]}"
+
+# resolve $SOURCE until the file is no longer a symlink
+while [[ -h $source ]]; do
+  scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+  source="$(readlink "$source")"
+
+  # if $source was a relative symlink, we need to resolve it relative to the path where the
+  # symlink file was located
+  [[ $source != /* ]] && source="$scriptroot/$source"
+done
+
+scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
+
+. "$scriptroot/install-nuget-credprovider.sh"
+
+"$scriptroot/common/msbuild.sh" "$@"

eng/install-nuget-credprovider.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+set -e
+
+# This script installs the NuGet Credential Provider.
+
+# Install curl if necessary. Dependency exists inside downloaded script.
+if command -v curl > /dev/null; then
+  echo "curl found."
+else
+  echo "curl not found, trying to install..."
+  (
+    set +e
+    set -x
+    apt update && apt install -y curl
+    apk update && apk upgrade && apk add curl
+    exit 0
+  )
+fi
+
+# Install. Ported from https://gist.github.com/shubham90/ad85f2546a72caa20d57bce03ec3890f
+install_credprovider() {
+  # Download the provider and install.
+  cred_provider_url='https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh'
+  curl "$cred_provider_url" -s -S -L | bash
+
+  # Environment variable to enable session token cache. More on this here: https://github.com/Microsoft/artifacts-credprovider#help
+  export NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true
+}
+
+install_credprovider
+
+# Additional setup to try to avoid flakiness: https://github.com/dotnet/arcade/issues/3932
+export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0
+export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20

eng/jobs/bash-build.yml

@@ -31,18 +31,32 @@ jobs:
           queue: buildpool.ubuntu.1604.amd64
     strategy: ${{ parameters.strategy }}
     variables:
+      # Preserve the NuGet authentication env vars into the Docker container.
+      # The 'NuGetAuthenticate' build step may have set these.
+      PreserveNuGetAuthDockerArgs: >-
+        -e VSS_NUGET_URI_PREFIXES
+        -e VSS_NUGET_ACCESSTOKEN
 
       ${{ if ne(parameters.name, 'FreeBSD_x64')}}:
         RunArguments: >-
           docker run --privileged --rm
           -v "$(Build.SourcesDirectory):/root/coresetup"
           -w="/root/coresetup"
+          $(PreserveNuGetAuthDockerArgs)
           ${{ parameters.additionalRunArgs }}
           ${{ parameters.dockerImage }}
       
       ${{ if eq(parameters.name, 'FreeBSD_x64')}}:
         RunArguments: export DotNetBootstrapCliTarPath=/dotnet-sdk-freebsd-x64.tar &&
 
+      ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+        BuildScript: ./eng/install-nuget-credprovider-then-build.sh
+        MSBuildScript: /root/coresetup/eng/install-nuget-credprovider-then-msbuild.sh
+
+      ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
+        BuildScript: ./build.sh
+        MSBuildScript: /root/coresetup/eng/common/msbuild.sh
+
       CommonMSBuildArgs: >-
         /p:Configuration=$(_BuildConfig)
         /p:OfficialBuildId=$(OfficialBuildId)
@@ -73,15 +87,18 @@ jobs:
         /p:SharedFrameworkPublishDir=/root/sharedFrameworkPublish/
         /p:InstallerSourceOSPlatformConfig=linux-x64.$(_BuildConfig)
 
-      MSBuildScript: /root/coresetup/eng/common/msbuild.sh
       DockerRunMSBuild: >-
         docker run
         -v $(Build.SourcesDirectory):/root/coresetup
         -v $(Build.StagingDirectory)/sharedFrameworkPublish/:/root/sharedFrameworkPublish/
         -w=/root/coresetup
+        $(PreserveNuGetAuthDockerArgs)
 
     steps:
 
+    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: NuGetAuthenticate@0
+
     # Builds don't set user ID, so files might be owned by root and unable to be cleaned up by AzDO.
     # Clean up the build dirs ourselves in another Docker container to avoid failures.
     # Using hosted agents is tracked by https://github.com/dotnet/core-setup/issues/4997
@@ -101,7 +118,7 @@ jobs:
     - script: |
         set -x
         df -h
-        $(RunArguments) ./build.sh $(BuildArguments)
+        $(RunArguments) $(BuildScript) $(BuildArguments)
       displayName: Build
 
     # Only for glibc leg, here we produce RPMs and Debs

eng/jobs/osx-build.yml

@@ -21,6 +21,10 @@ jobs:
       /p:Configuration=$(_BuildConfig)
       /p:PortableBuild=true
   steps:
+
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: NuGetAuthenticate@0
+
   - script: >-
       $(Build.SourcesDirectory)/build.sh --ci --test
       /p:OfficialBuildId=$(OfficialBuildId)

eng/jobs/prepare-signed-artifacts.yml

@@ -16,6 +16,9 @@ jobs:
 
   steps:
 
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: NuGetAuthenticate@0
+
   - task: MicroBuildSigningPlugin@2
     displayName: Install MicroBuild plugin for Signing
     inputs:

eng/jobs/run-publish-project.yml

@@ -25,23 +25,76 @@ jobs:
   # Only get the secret variable groups if the def has the official name. Reduce dev build def risk.
   - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}:
     # Used for publishing individual leg assets to azure blob storage
-    - group: DotNet-DotNetCli-Storage
+    - ${{ if eq(parameters.dependency.channel.public, 'true') }}:
+      - group: DotNet-DotNetCli-Storage
+    - ${{ if ne(parameters.dependency.channel.public, 'true') }}:
+      - group: DotNet-MSRC-Storage
     # Used for dotnet/versions update
     - group: DotNet-Versions-Publish
 
-    # Blob storage publish (installers and checksums)
+  - name: _DefaultContainerName
+    value: dotnet
+  - name: _DefaultChecksumsContainerName
+    value: dotnet
+
+  - ${{ if eq(parameters.dependency.channel.public, 'true') }}:
+    - name: _DefaultAzureAccountName
+      value: dotnetcli
+    - name: _DefaultAzureAccessToken
+      value: $(dotnetcli-storage-key)
+    - name: _DefaultChecksumAzureAccountName
+      value: dotnetclichecksums
+    - name: _DefaultChecksumAzureAccessToken
+      value: $(dotnetclichecksums-storage-key)
+    # dotnet/versions update
+    - name: _GitHubUser
+      value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ]
+    - name: _GitHubEmail
+      value: $[ coalesce(variables.GitHubEmail, '[email protected]') ]
+    - name: _GitHubAuthToken
+      value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ]
+    - name: _VersionsRepoOwner
+      value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ]
+    - name: _VersionsRepo
+      value: $[ coalesce(variables.VersionsRepo, 'versions') ]
+    - name: _DotNetVersionsArgs
+      value: >-
+        /p:GitHubUser=$(_GitHubUser)
+        /p:GitHubEmail=$(_GitHubEmail)
+        /p:GitHubAuthToken=$(_GitHubAuthToken)
+        /p:VersionsRepoOwner=$(_VersionsRepoOwner)
+        /p:VersionsRepo=$(_VersionsRepo)
+        /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName)
+
+  - ${{ if ne(parameters.dependency.channel.public, 'true') }}:
+    - name: _DefaultAzureAccountName
+      value: dotnetclimsrc
+    - name: _DefaultAzureAccessToken
+      value: $(dotnetclimsrc-access-key)
+    - name: _DefaultChecksumAzureAccountName
+      value: dotnetclimsrc
+    - name: _DefaultChecksumsContainerName
+      value: dotnet-checksums
+    - name: _DefaultChecksumAzureAccessToken
+      value: $(dotnetclimsrc-access-key)
+    # dotnet/versions update (disabled)
+    - name: _DotNetVersionsArgs
+      value: ''
+
+  # Blob storage publish (installers and checksums)
   - name: _AzureAccountName
-    value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ]
+    value: $[ coalesce(variables.AzureAccountName, '$(_DefaultAzureAccountName)') ]
   - name: _ContainerName
-    value: $[ coalesce(variables.ContainerName, 'dotnet') ]
+    value: $[ coalesce(variables.ContainerName, '$(_DefaultContainerName)') ]
   - name: _AzureAccessToken
-    value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ]
+    value: $[ coalesce(variables.AzureAccessToken, '$(_DefaultAzureAccessToken)') ]
   - name: _ChecksumAzureAccountName
-    value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ]
+    value: $[ coalesce(variables.ChecksumAzureAccountName, '$(_DefaultChecksumAzureAccountName)') ]
   - name: _ChecksumContainerName
-    value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ]
+    value: $[ coalesce(variables.ChecksumContainerName, '$(_DefaultChecksumsContainerName)') ]
   - name: _ChecksumAzureAccessToken
-    value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ]
+    value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(_DefaultChecksumAzureAccessToken)') ]
+
   - name: _CommonPublishArgs
     value: >-
       /p:AzureAccountName=$(_AzureAccountName)
@@ -51,28 +104,11 @@ jobs:
       /p:ChecksumContainerName=$(_ChecksumContainerName)
       /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken)
 
-  # dotnet/versions update
-  - name: _GitHubUser
-    value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ]
-  - name: _GitHubEmail
-    value: $[ coalesce(variables.GitHubEmail, '[email protected]') ]
-  - name: _GitHubAuthToken
-    value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ]
-  - name: _VersionsRepoOwner
-    value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ]
-  - name: _VersionsRepo
-    value: $[ coalesce(variables.VersionsRepo, 'versions') ]
-  - name: _DotNetVersionsArgs
-    value: >-
-      /p:GitHubUser=$(_GitHubUser)
-      /p:GitHubEmail=$(_GitHubEmail)
-      /p:GitHubAuthToken=$(_GitHubAuthToken)
-      /p:VersionsRepoOwner=$(_VersionsRepoOwner)
-      /p:VersionsRepo=$(_VersionsRepo)
-      /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName)
-
   steps:
 
+  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+    - task: NuGetAuthenticate@0
+
   - task: DownloadBuildArtifacts@0
     displayName: Download Artifacts
     inputs:

eng/jobs/steps/build-linux-package.yml

@@ -21,7 +21,7 @@ steps:
 - script: |
     set -x
     df -h
-    $(DockerRunMSBuild) ${{ parameters.image }} ./build.sh \
+    $(DockerRunMSBuild) ${{ parameters.image }} $(BuildScript) \
       --ci \
       /p:Subset=Installer \
       /p:UsePrebuiltPortableBinariesForInstallers=true \

eng/jobs/windows-build.yml

@@ -41,6 +41,8 @@ jobs:
     steps:
 
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      - task: NuGetAuthenticate@0
+
       - task: MicroBuildSigningPlugin@2
         displayName: Install MicroBuild plugin for Signing
         inputs:

eng/pipelines/installer/azure-pipelines.yml

@@ -183,28 +183,50 @@ stages:
       #     bar: <Name of the Arcade variable that contains the ID of this channel in BAR>
       #     storage: <Name of the Latest channel to publish to in dotnetcli blob storage>
       dependsOnPublishStages:
+
       - dependsOn: NetCore_Dev30_Publish
         channel:
           name: .NET Core 3 Dev
           bar: PublicDevRelease_30_Channel_Id
           storage: release/3.0
+          public: true
+      - dependsOn: NetCore_Release30_Publish
+        channel:
+          name: .NET Core 3 Release
+          bar: PublicRelease_30_Channel_Id
+          storage: release/3.0-preview9
+          public: true
+      - dependsOn: NetCore_30_Internal_Servicing_Publish
+        channel:
+          name: .NET Core 3 Internal Servicing
+          bar: InternalServicing_30_Channel_Id
+          storage: internal/release/3.0
+          public: false
+
       - dependsOn: NetCore_Dev31_Publish
         channel:
           name: .NET Core 3.1 Dev
           bar: PublicDevRelease_31_Channel_Id
           storage: release/3.1
+          public: true
+
       - dependsOn: NetCore_Release31_Publish
         channel:
           name: .NET Core 3.1 Release
           bar: PublicRelease_31_Channel_Id
           storage: release/3.1-preview1
+          public: true
+
       - dependsOn: NetCore_Dev5_Publish
         channel:
           name: .NET Core 5 Dev
           bar: NetCore_5_Dev_Channel_Id
           storage: master
+          public: true
+
       - dependsOn: PVR_Publish
         channel:
           name: .NET Tools - Validation
           bar: PublicValidationRelease_30_Channel_Id
           storage: dev/validation
+          public: true

src/installer/test/Directory.Build.props

@@ -7,6 +7,7 @@
     <TestStabilizedLegacyPackagesDir>$(ObjDir)TestStabilizedPackages\</TestStabilizedLegacyPackagesDir>
     <TestRestorePackagesPath>$(ObjDir)TestPackageCache\</TestRestorePackagesPath>
     <TestRestoreNuGetConfigFile>$(ObjDir)TestNuGetConfig\NuGet.config</TestRestoreNuGetConfigFile>
+    <InternalNupkgCacheDir>$(ObjDir)ExtraNupkgsForTestRestore\</InternalNupkgCacheDir>
     <TestArchitectures>$(TargetArchitecture)</TestArchitectures>
     <TestInfraTargetFramework>netcoreapp3.0</TestInfraTargetFramework>
   </PropertyGroup>