Build HttpStress and SslStress with live-built runtime using current …

…TFM (dotnet#61689)

This PR changes both local (non-containerized) and containerized stress builds to build against the live-built runtime with the help of targetingpacks.targets.

.gitignore

@@ -354,3 +354,7 @@ src/coreclr/System.Private.CoreLib/common
 !src/coreclr/inc/obj/
 !src/coreclr/vm/.vscode/
 !src/coreclr/vm/.vscode/c_cpp_properties.json
+
+# Temporary artifacts from local libraries stress builds
+.dotnet-daily/
+run-stress-*

eng/docker/build-docker-sdk.ps1

@@ -6,21 +6,16 @@
 Param(
   [string][Alias('t')]$imageName = "dotnet-sdk-libs-current",
   [string][Alias('c')]$configuration = "Release",
-  [switch][Alias('w')]$buildWindowsContainers,
-  [switch][Alias('pa')]$privateAspNetCore
+  [switch][Alias('w')]$buildWindowsContainers
 )
 
+$dotNetVersion="7.0"
 $ErrorActionPreference = "Stop"
 
 $REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
 
 $dockerFilePrefix="$PSScriptRoot/libraries-sdk"
 
-if ($privateAspNetCore)
-{
-  $dockerFilePrefix="$PSScriptRoot/libraries-sdk-aspnetcore"
-}
-
 if ($buildWindowsContainers)
 {
   # Due to size concerns, we don't currently do docker builds on windows.
@@ -34,12 +29,39 @@ if ($buildWindowsContainers)
   }
 
   $dockerFile="$dockerFilePrefix.windows.Dockerfile"
+
+  # Collect the following artifacts to folder, that will be used as build context for the container,
+  # so projects can build and test against the live-built runtime:
+  # 1. Reference assembly pack (microsoft.netcore.app.ref)
+  # 2. Runtime pack (microsoft.netcore.app.runtime.win-x64)
+  # 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+  # 4. testhost
+  $binArtifacts = "$REPO_ROOT_DIR\artifacts\bin"
+  $dockerContext = "$REPO_ROOT_DIR\artifacts\docker-context"
 
+  if (Test-Path $dockerContext) {
+      Remove-Item -Recurse -Force $dockerContext
+  }
+
+  Copy-Item -Recurse -Path $binArtifacts\microsoft.netcore.app.ref `
+                     -Destination $dockerContext\microsoft.netcore.app.ref
+  Copy-Item -Recurse -Path $binArtifacts\microsoft.netcore.app.runtime.win-x64 `
+                     -Destination $dockerContext\microsoft.netcore.app.runtime.win-x64
+  Copy-Item -Recurse -Path $binArtifacts\testhost `
+                     -Destination $dockerContext\testhost
+  Copy-Item -Recurse -Path $REPO_ROOT_DIR\eng\targetingpacks.targets `
+                     -Destination $dockerContext\targetingpacks.targets
+
+  # In case of non-CI builds, testhost may already contain Microsoft.AspNetCore.App (see build-local.ps1 in HttpStress):
+  $testHostAspNetCorePath="$dockerContext\testhost\net$dotNetVersion-windows-$configuration-x64/shared/Microsoft.AspNetCore.App"
+  if (Test-Path $testHostAspNetCorePath) {
+    Remove-Item -Recurse -Force $testHostAspNetCorePath
+  }
+
   docker build --tag $imageName `
     --build-arg CONFIGURATION=$configuration `
-    --build-arg TESTHOST_LOCATION=. `
     --file $dockerFile `
-    "$REPO_ROOT_DIR/artifacts/bin/testhost"
+    $dockerContext
 }
 else
 {

eng/docker/build-docker-sdk.sh

@@ -23,7 +23,6 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 
 imagename="dotnet-sdk-libs-current"
 configuration="Release"
-privateaspnetcore=0
 
 while [[ $# > 0 ]]; do
   opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")"
@@ -36,10 +35,6 @@ while [[ $# > 0 ]]; do
       configuration=$2
       shift 2
       ;;
-    -privateaspnetcore|-pa)
-      privateaspnetcore=1
-      shift 1
-      ;;
     *)
       shift 1
       ;;
@@ -49,13 +44,9 @@ done
 repo_root=$(git rev-parse --show-toplevel)
 docker_file="$scriptroot/libraries-sdk.linux.Dockerfile"
 
-if [[ $privateaspnetcore -eq 1 ]]; then
-    docker_file="$scriptroot/libraries-sdk-aspnetcore.linux.Dockerfile"
-fi
-
 docker build --tag $imagename \
     --build-arg CONFIGURATION=$configuration \
     --file $docker_file \
     $repo_root
 
-exit $?
+exit $?

eng/docker/libraries-sdk.linux.Dockerfile

@@ -4,25 +4,47 @@ ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 
 FROM $BUILD_BASE_IMAGE as corefxbuild
 
+ARG CONFIGURATION=Release
+
 WORKDIR /repo
 COPY . .
-
-ARG CONFIGURATION=Release
-RUN ./build.sh -ci -subset clr+libs -runtimeconfiguration release -c $CONFIGURATION
+RUN ./build.sh clr+libs -runtimeconfiguration Release -configuration $CONFIGURATION -ci
 
 FROM $SDK_BASE_IMAGE as target
 
-ARG TESTHOST_LOCATION=/repo/artifacts/bin/testhost
-ARG TFM=net7.0
-ARG OS=Linux
-ARG ARCH=x64
+ARG VERSION=7.0
 ARG CONFIGURATION=Release
+ENV _DOTNET_INSTALL_CHANNEL="$VERSION.1xx"
+
+# Install latest daily SDK:
+RUN wget https://dot.net/v1/dotnet-install.sh
+RUN bash ./dotnet-install.sh --channel $_DOTNET_INSTALL_CHANNEL --quality daily --install-dir /usr/share/dotnet
+
+# Collect the following artifacts under /live-runtime-artifacts,
+# so projects can build and test against the live-built runtime:
+# 1. Reference assembly pack (microsoft.netcore.app.ref)
+# 2. Runtime pack (microsoft.netcore.app.runtime.linux-x64)
+# 3. targetingpacks.targets, so stress test builds can target the live-built runtime instead of the one in the pre-installed SDK
+# 4. testhost
 
-ARG COREFX_SHARED_FRAMEWORK_NAME=Microsoft.NETCore.App
-ARG SOURCE_COREFX_VERSION=7.0.0
-ARG TARGET_SHARED_FRAMEWORK=/usr/share/dotnet/shared
-ARG TARGET_COREFX_VERSION=$DOTNET_VERSION
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/microsoft.netcore.app.ref \
+    /live-runtime-artifacts/microsoft.netcore.app.ref
+
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/microsoft.netcore.app.runtime.linux-x64 \
+    /live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64
 
 COPY --from=corefxbuild \
-    $TESTHOST_LOCATION/$TFM-$OS-$CONFIGURATION-$ARCH/shared/$COREFX_SHARED_FRAMEWORK_NAME/$SOURCE_COREFX_VERSION/* \
-    $TARGET_SHARED_FRAMEWORK/$COREFX_SHARED_FRAMEWORK_NAME/$TARGET_COREFX_VERSION/
+    /repo/eng/targetingpacks.targets \
+    /live-runtime-artifacts/targetingpacks.targets
+
+COPY --from=corefxbuild \
+    /repo/artifacts/bin/testhost \
+    /live-runtime-artifacts/testhost
+
+# Add AspNetCore bits to testhost:
+ENV _ASPNETCORE_SOURCE="/usr/share/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+ENV _ASPNETCORE_DEST="/live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
+RUN mkdir -p $_ASPNETCORE_DEST
+RUN cp -r $_ASPNETCORE_SOURCE $_ASPNETCORE_DEST

eng/docker/libraries-sdk.windows.Dockerfile

@@ -3,17 +3,23 @@
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-nanoserver-1809
 FROM $SDK_BASE_IMAGE as target
 
-ARG TESTHOST_LOCATION=".\\artifacts\\bin\\testhost"
-ARG TFM=net7.0
-ARG OS=windows
-ARG ARCH=x64
+SHELL ["pwsh", "-Command", "$ErrorActionPreference = 'Stop'; $ProgressPreference = 'SilentlyContinue';"]
+
+ARG VERSION=7.0
+ENV _DOTNET_INSTALL_CHANNEL="$VERSION.1xx"
 ARG CONFIGURATION=Release
 
-ARG COREFX_SHARED_FRAMEWORK_NAME=Microsoft.NETCore.App
-ARG SOURCE_COREFX_VERSION=7.0.0
-ARG TARGET_SHARED_FRAMEWORK="C:\\Program Files\\dotnet\\shared"
-ARG TARGET_COREFX_VERSION=$DOTNET_VERSION
+USER ContainerAdministrator
+
+RUN Invoke-WebRequest -Uri https://dot.net/v1/dotnet-install.ps1 -OutFile .\dotnet-install.ps1
+RUN & .\dotnet-install.ps1 -Channel $env:_DOTNET_INSTALL_CHANNEL -Quality daily -InstallDir 'C:/Program Files/dotnet'
+
+USER ContainerUser
+
+COPY . /live-runtime-artifacts
 
-COPY `
-    $TESTHOST_LOCATION\$TFM-$OS-$CONFIGURATION-$ARCH\shared\$COREFX_SHARED_FRAMEWORK_NAME\$SOURCE_COREFX_VERSION\ `
-    $TARGET_SHARED_FRAMEWORK\$COREFX_SHARED_FRAMEWORK_NAME\$TARGET_COREFX_VERSION\
+# Add AspNetCore bits to testhost:
+ENV _ASPNETCORE_SOURCE="C:/Program Files/dotnet/shared/Microsoft.AspNetCore.App/$VERSION*"
+ENV _ASPNETCORE_DEST="C:/live-runtime-artifacts/testhost/net$VERSION-windows-$CONFIGURATION-x64/shared/Microsoft.AspNetCore.App"
+RUN & New-Item -ItemType Directory -Path $env:_ASPNETCORE_DEST
+RUN Copy-Item -Recurse -Path $env:_ASPNETCORE_SOURCE -Destination $env:_ASPNETCORE_DEST

eng/pipelines/libraries/stress/http.yml

@@ -57,6 +57,7 @@ jobs:
       export HTTPSTRESS_CLIENT_ARGS="$HTTPSTRESS_CLIENT_ARGS -http 3.0"
       export HTTPSTRESS_SERVER_ARGS="$HTTPSTRESS_SERVER_ARGS -http 3.0"
       docker-compose up --abort-on-container-exit --no-color
+    timeoutInMinutes: 35 # In case the HTTP/3.0 run hangs, we timeout shortly after the expected 30 minute run
     displayName: Run HttpStress - HTTP 3.0
     condition: and(eq(variables['buildRuntime.succeeded'], 'true'), eq(variables['buildStress.succeeded'], 'true'))
 
@@ -149,4 +150,4 @@ jobs:
   - powershell: |
       Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True
     name: enableFirewall
-    displayName: Enable Firewall
+    displayName: Enable Firewall

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.props

@@ -1 +1,17 @@
-<Project/>
+<Project>
+    <PropertyGroup>
+        <PackageRid>linux-x64</PackageRid>
+        <PackageRid Condition="$([MSBuild]::IsOSPlatform('WINDOWS'))">win-x64</PackageRid>
+
+        <!-- Stress projects have their own global.json, the directory above that also has it is the repository root. -->
+        <RepositoryRoot>$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory)../, global.json))/</RepositoryRoot>
+
+        <TargetingPacksTargetsLocation Condition="'$(TargetingPacksTargetsLocation)' == ''">$(RepositoryRoot)eng/targetingpacks.targets</TargetingPacksTargetsLocation>
+        <ProductVersion>7.0.0</ProductVersion>
+        <NetCoreAppCurrent>net7.0</NetCoreAppCurrent>
+        <NetCoreAppCurrentVersion>7.0</NetCoreAppCurrentVersion>
+        <MicrosoftNetCoreAppFrameworkName>Microsoft.NETCore.App</MicrosoftNetCoreAppFrameworkName>
+        <MicrosoftNetCoreAppRefPackDir Condition="'$(MicrosoftNetCoreAppRefPackDir)' == ''" >$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.ref/</MicrosoftNetCoreAppRefPackDir>
+        <MicrosoftNetCoreAppRuntimePackDir Condition="'$(MicrosoftNetCoreAppRuntimePackDir)' == ''">$(RepositoryRoot)artifacts/bin/microsoft.netcore.app.runtime.$(PackageRid)/$(Configuration)/</MicrosoftNetCoreAppRuntimePackDir>
+    </PropertyGroup>
+</Project>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Directory.Build.targets

@@ -1 +1,11 @@
-<Project/>
+<Project>
+  <Import Project="$(TargetingPacksTargetsLocation)" />
+
+  <PropertyGroup>
+    <!--
+      Define this here because the SDK resets it
+      unconditionally in Microsoft.NETCoreSdk.BundledVersions.props.
+    -->
+    <NETCoreAppMaximumVersion>7.0</NETCoreAppMaximumVersion>
+  </PropertyGroup>
+</Project>

src/libraries/System.Net.Http/tests/StressTests/HttpStress/Dockerfile

@@ -1,9 +1,6 @@
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/nightly/sdk:6.0-bullseye-slim
 FROM $SDK_BASE_IMAGE
 
-RUN echo "DOTNET_SDK_VERSION="$DOTNET_SDK_VERSION
-RUN echo "DOTNET_VERSION="$DOTNET_VERSION
-
 WORKDIR /app
 COPY . .
 
@@ -19,8 +16,13 @@ RUN unzip $PACKAGES_DIR.zip
 RUN dpkg -i $PACKAGES_DIR/$MSQUIC_PACKAGE
 RUN rm -rf $PACKAGES_DIR*
 
+ARG VERSION=7.0
 ARG CONFIGURATION=Release
-RUN dotnet build -c $CONFIGURATION
+
+RUN dotnet build -c $CONFIGURATION \
+    -p:TargetingPacksTargetsLocation=/live-runtime-artifacts/targetingpacks.targets \
+    -p:MicrosoftNetCoreAppRefPackDir=/live-runtime-artifacts/microsoft.netcore.app.ref/ \
+    -p:MicrosoftNetCoreAppRuntimePackDir=/live-runtime-artifacts/microsoft.netcore.app.runtime.linux-x64/$CONFIGURATION/
 
 # Enable dump collection
 ENV COMPlus_DbgEnableMiniDump=1
@@ -29,6 +31,8 @@ ENV COMPlus_DbgMiniDumpName="/share/coredump.%p"
 
 EXPOSE 5001
 
+ENV VERSION=$VERSION
 ENV CONFIGURATION=$CONFIGURATION
 ENV HTTPSTRESS_ARGS=''
-CMD dotnet run --no-build -c $CONFIGURATION -- $HTTPSTRESS_ARGS
+CMD /live-runtime-artifacts/testhost/net$VERSION-Linux-$CONFIGURATION-x64/dotnet exec \
+    ./bin/$CONFIGURATION/net$VERSION/HttpStress.dll $HTTPSTRESS_ARGS

src/libraries/System.Net.Http/tests/StressTests/HttpStress/HttpStress.csproj

@@ -1,8 +1,7 @@
-<Project Sdk="Microsoft.NET.Sdk.Web">
-
+<Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net6.0</TargetFramework>
+    <TargetFramework>$(NetCoreAppCurrent)</TargetFramework>
     <LangVersion>preview</LangVersion>
     <Nullable>enable</Nullable>
     <EnablePreviewFeatures>True</EnablePreviewFeatures>
@@ -22,4 +21,16 @@
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Quic" Version="6.0.0-preview.5.21301.17" />
   </ItemGroup>
 
-</Project>
+  <PropertyGroup>
+    <!-- These may lead to duplicate generated classes with local (non-docker) Linux builds. -->
+    <GenerateAssemblyCompanyAttribute>false</GenerateAssemblyCompanyAttribute>
+    <GenerateAssemblyFileVersionAttribute>false</GenerateAssemblyFileVersionAttribute>
+    <GenerateAssemblyInformationalVersionAttribute>false</GenerateAssemblyInformationalVersionAttribute>
+    <GenerateAssemblyVersionAttribute>false</GenerateAssemblyVersionAttribute>
+    <GenerateAssemblyConfigurationAttribute>false</GenerateAssemblyConfigurationAttribute>
+    <GenerateAssemblyProductAttribute>false</GenerateAssemblyProductAttribute>
+    <GenerateAssemblyTitleAttribute>false</GenerateAssemblyTitleAttribute>
+    <GenerateAssemblyInfo>false</GenerateAssemblyInfo>
+    <GenerateTargetFrameworkAttribute>false</GenerateTargetFrameworkAttribute>
+  </PropertyGroup>
+</Project>