Consolidate NCL stress testing infrastructure (dotnet#700)

eng/docker/Readme.md

@@ -0,0 +1,30 @@
+# Docker Build Infrastructure
+
+Provides reusable docker build infrastructure for the dotnet/runtime repo.
+
+## libraries-sdk Dockerfiles
+
+The `libraries-sdk` Dockerfiles can be used to build dotnet sdk docker images
+that contain the current libraries built from source. 
+These images can be used to build dockerized dotnet services that target the current libraries.
+Currently, debian and windows nanoserver sdk's are supported.
+
+### Building the images
+
+To build the linux image locally
+
+```powershell
+PS> .\build-docker-sdk.ps1 -t dotnet-linux-sdk-current
+```
+
+and for Windows:
+
+```powershell
+PS> .\build-socker-sdk.ps1 -w -t dotnet-nanoserver-sdk-current
+```
+
+To use Debug builds:
+
+```powershell
+PS> .\build-docker-sdk.ps1 -c Debug -t dotnet-sdk-current
+```

eng/docker/build-docker-sdk.ps1

@@ -0,0 +1,35 @@
+#!/usr/bin/env pwsh
+# Builds libraries and produces a dotnet sdk docker image
+# that contains the current bits in its shared framework folder.
+
+[CmdletBinding(PositionalBinding=$false)]
+Param(
+  [string][Alias('t')]$imageName = "dotnet-sdk-libs-current",
+  [string][Alias('c')]$configuration = "Release",
+  [switch][Alias('w')]$buildWindowsContainers
+)
+
+$REPO_ROOT_DIR=$(git -C "$PSScriptRoot" rev-parse --show-toplevel)
+
+if ($buildWindowsContainers)
+{
+  # Due to size concerns, we don't currently do docker builds on windows.
+  # Build on the host machine, then simply copy artifacts to the target docker image.
+  # This should result in significantly lower build times, for now.
+  & "$REPO_ROOT_DIR/libraries.cmd" -ci -c $configuration
+
+  # Dockerize the build artifacts
+  docker build --tag $imageName `
+      --build-arg CONFIGURATION=$configuration `
+      --build-arg TESTHOST_LOCATION=. `
+      --file "$PSScriptRoot/libraries-sdk.windows.Dockerfile" `
+      "$REPO_ROOT_DIR/artifacts/bin/testhost"
+}
+else 
+{
+  # Docker build libraries and copy to dotnet sdk image
+  docker build --tag $imageName `
+      --build-arg CONFIGURATION=$configuration `
+      --file "$PSScriptRoot/libraries-sdk.linux.Dockerfile" `
+      $REPO_ROOT_DIR
+}

src/libraries/System.Net.Http/tests/StressTests/HttpStress/corefx.Dockerfile → eng/docker/libraries-sdk.linux.Dockerfile

@@ -1,5 +1,5 @@
-# Builds a dotnet sdk base image which contains corefx bits compiled from source
-ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:rhel7_prereqs_2
+# Builds and copies library artifacts into target dotnet sdk image
+ARG BUILD_BASE_IMAGE=mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-f39df28-20191023143754
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/core/sdk:3.0.100-buster
 
 FROM $BUILD_BASE_IMAGE as corefxbuild
@@ -9,7 +9,7 @@ COPY . .
 
 ARG CONFIGURATION=Release
 ARG BUILD_SCRIPT_NAME=libraries
-RUN ./$BUILD_SCRIPT_NAME.sh -c $CONFIGURATION
+RUN ./libraries.sh -c $CONFIGURATION
 
 FROM $SDK_BASE_IMAGE as target
 

src/libraries/System.Net.Http/tests/StressTests/HttpStress/corefx.windows.Dockerfile → eng/docker/libraries-sdk.windows.Dockerfile

@@ -1,8 +1,6 @@
 # escape=`
+# Simple Dockerfile which copies library build artifacts into target dotnet sdk image
 ARG SDK_BASE_IMAGE=mcr.microsoft.com/dotnet/core/sdk:3.0.100-nanoserver-1809
-
-# Simple Dockerfile which copies testhost shared framework artifacts into a target dotnet sdk image
-
 FROM $SDK_BASE_IMAGE as target
 
 ARG TESTHOST_LOCATION=".\\artifacts\\bin\\testhost"

eng/pipelines/libraries/stress/http.yml

@@ -0,0 +1,72 @@
+trigger: none
+
+pr:
+  branches:
+    include:
+    - "*"
+
+schedules:
+- cron: "0 13 * * *" # 1PM UTC => 5 AM PST
+  displayName: HttpStress nightly run
+  branches:
+    include:
+    - master
+
+variables:
+  - template: ../variables.yml
+  - name: dockerfilesFolder
+    value: $(Build.SourcesDirectory)/eng/docker
+  - name: httpStressProject
+    value: $(sourcesRoot)/System.Net.Http/tests/StressTests/HttpStress
+  - name: sdkBaseImage
+    value: dotnet-sdk-libraries-current
+
+
+jobs:
+
+- job: linux
+  displayName: Docker Linux
+  pool:
+    name: Hosted Ubuntu 1604
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchDepth: 5
+
+  - bash: |
+      $(dockerfilesFolder)/build-docker-sdk.ps1 -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION)
+    displayName: Build Libraries
+  
+  - bash: |
+      $(httpStressProject)/run-docker-compose.ps1 -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage)
+    displayName: Build HttpStress
+  
+  - bash: |
+      cd '$(httpStressProject)'
+      docker-compose up --abort-on-container-exit --no-color
+    displayName: Run HttpStress
+
+- job: windows
+  displayName: Docker Nano Server
+  pool:
+    vmImage: 'windows-latest'
+
+  steps:
+  - checkout: self
+    clean: true
+    fetchDepth: 1
+    lfs: false
+
+  - pwsh: |
+      $(dockerfilesFolder)/build-docker-sdk.ps1 -w -t $(sdkBaseImage) -c $(BUILD_CONFIGURATION)
+    displayName: Build Libraries
+  
+  - pwsh: |
+      $(httpStressProject)/run-docker-compose.ps1 -w -o -c $(BUILD_CONFIGURATION) -t $(sdkBaseImage)
+    displayName: Build HttpStress
+  
+  - pwsh: |
+      cd '$(httpStressProject)'
+      docker-compose up --abort-on-container-exit --no-color
+    displayName: Run HttpStress