Add multi system test platform infrastructure (microsoft#573)

Perf Tests can now run either locally or cross systems. A setup has been created to make this work automatically on AZP. Additionally, machine name is now published, along with other smaller code changes.
AliceYeh · Jul 9, 2020 · 6e1c47a · 6e1c47a
1 parent aaeb595
commit 6e1c47a
Show file tree

Hide file tree

Showing 13 changed files with 845 additions and 426 deletions.
diff --git a/.azure/azure-pipelines.ci.yml b/.azure/azure-pipelines.ci.yml
@@ -213,28 +213,49 @@ stages:
     parameters:
       pool: MsQuic-Win-Perf
       platform: windows
-      tls: stub
+      localTls: schannel
+      remoteTls: schannel
+      extraArgs: '-Publish'
   - template: ./templates/run-performance.yml
     parameters:
       pool: MsQuic-Win-Perf
       platform: windows
-      tls: schannel
+      localTls: schannel
+      remoteTls: schannel
+      extraName: 'Loopback'
+      extraArgs: '-Publish -Local'
   - template: ./templates/run-performance.yml
     parameters:
       pool: MsQuic-Win-Perf
       platform: windows
-      tls: mitls
+      localTls: stub
+      remoteTls: stub
+      extraName: 'Loopback'
+      extraArgs: '-Publish -Local'
+  - template: ./templates/run-performance.yml
+    parameters:
+      pool: MsQuic-Win-Perf
+      platform: windows
+      localTls: mitls
+      remoteTls: mitls
+      extraName: 'Loopback'
+      extraArgs: '-Publish -Local'
   - template: ./templates/run-performance.yml
     parameters:
       image: ubuntu-latest
       platform: linux
-      tls: openssl
+      localTls: openssl
+      remoteTls: openssl
+      extraName: 'Loopback'
+      extraArgs: '-Publish -Local'
   - template: ./templates/run-performance.yml
     parameters:
       image: ubuntu-latest
       platform: linux
-      tls: stub
-  - template: ./templates/publish-performance.yml
+      localTls: stub
+      remoteTls: stub
+      extraName: 'Loopback'
+      extraArgs: '-Publish -Local'
 
 #
 # QuicInterop

diff --git a/.azure/azure-pipelines.perf.yml b/.azure/azure-pipelines.perf.yml
@@ -67,18 +67,25 @@ stages:
     parameters:
       pool: MsQuic-Win-Perf
       platform: windows
-      tls: schannel
+      localTls: schannel
+      remoteTls: schannel
       ${{ if eq(parameters.mode, 'PGO') }}:
-        extraArgs: -PGO
+        extraArgs: -PGO -Local
       ${{ if eq(parameters.mode, 'Record') }}:
-        extraArgs: -Record
+        extraArgs: -Record -Local
+      ${{ if eq(parameters.mode, 'Normal') }}:
+        extraArgs: -Local
+
   - template: ./templates/run-performance.yml
     parameters:
       pool: MsQuic-Win-Perf
       platform: windows
-      tls: schannel
+      localTls: schannel
+      remoteTls: schannel
       arch: x86
       ${{ if eq(parameters.mode, 'PGO') }}:
-        extraArgs: -PGO
+        extraArgs: -PGO -Local
       ${{ if eq(parameters.mode, 'Record') }}:
-        extraArgs: -Record
+        extraArgs: -Record -Local
+      ${{ if eq(parameters.mode, 'Normal') }}:
+        extraArgs: -Local
diff --git a/.azure/templates/publish-performance.yml b/.azure/templates/publish-performance.yml
diff --git a/.azure/templates/run-performance.yml b/.azure/templates/run-performance.yml
@@ -6,12 +6,14 @@ parameters:
   platform: ''
   config: 'Release'
   arch: 'x64'
-  tls: ''
+  localTls: ''
+  remoteTls: ''
+  extraName: ''
   extraArgs: '-Publish'
 
 jobs:
-- job: performance_${{ parameters.platform }}_${{ parameters.arch }}_${{ parameters.tls }}
-  displayName: ${{ parameters.platform }} ${{ parameters.arch }} ${{ parameters.tls }}
+- job: performance_${{ parameters.platform }}_${{ parameters.arch }}_${{ parameters.localTls }}_${{ parameters.remoteTls }}_${{ parameters.extraName }}
+  displayName: ${{ parameters.platform }} ${{ parameters.arch }} ${{ parameters.localTls }} ${{ parameters.remoteTls }} ${{ parameters.extraName }}
   ${{ if ne(parameters.pool, '') }}:
     pool: ${{ parameters.pool }}
   ${{ if eq(parameters.pool, '') }}:
@@ -25,17 +27,23 @@ jobs:
   steps:
   - checkout: self
 
-  - ${{ if ne(parameters.pool, '') }}:
-    - task: DeleteFiles@1
-      displayName: Clear Old Artifacts
-      inputs:
-        contents: '**/artifacts/*'
+  - task: DeleteFiles@1
+    displayName: Clear Old Artifacts
+    inputs:
+      contents: '**/artifacts/*'
 
   - template: ./download-artifacts.yml
     parameters:
       platform: ${{ parameters.platform }}
       arch: ${{ parameters.arch }}
-      tls: ${{ parameters.tls }}
+      tls: ${{ parameters.localTls }}
+
+  - ${{ if ne(parameters.localTls, parameters.remoteTls) }}:
+    - template: ./download-artifacts.yml
+      parameters:
+        platform: ${{ parameters.platform }}
+        arch: ${{ parameters.arch }}
+        tls: ${{ parameters.remoteTls }}
 
   - task: PowerShell@2
     displayName: Prepare Test Machine
@@ -51,7 +59,7 @@ jobs:
     inputs:
       pwsh: true
       filePath: scripts/performance.ps1
-      arguments: -Config ${{ parameters.config }} -Arch ${{ parameters.arch }} -Tls ${{ parameters.tls }} ${{ parameters.extraArgs }}
+      arguments: -Config ${{ parameters.config }} -LocalTls ${{ parameters.localTls }} -RemoteTls ${{ parameters.remoteTls }} -LocalArch ${{ parameters.arch }} -RemoteArch ${{ parameters.arch }}  ${{ parameters.extraArgs }}
 
   - task: CopyFiles@2
     displayName: Move Performance Results
@@ -65,3 +73,12 @@ jobs:
       artifactName: performance
       pathToPublish: $(Build.ArtifactStagingDirectory)
       parallel: true
+
+  - task: Powershell@2
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    displayName: Publish Performance Results
+    inputs:
+      pwsh: true
+      filePath: scripts/publish-performance-results.ps1
+    env:
+      MAPPED_DEPLOYMENT_KEY: $(DatabaseDeploymentKey)
diff --git a/.gitignore b/.gitignore
@@ -364,3 +364,6 @@ FodyWeavers.xsd
 
 # VS Code settings folder
 .vscode/
+
+# PGO Local Test Copy
+artifacts_server/
diff --git a/scripts/RemoteTests.json b/scripts/RemoteTests.json
@@ -0,0 +1,58 @@
+[
+    {
+        "TestName": "Throughput",
+        "Remote" : {
+            "Platform": "Windows",
+            "Tls": ["stub", "schannel", "mitls"],
+            "Arch": ["x64", "x86", "arm", "arm64"],
+            "Exe": "quicping",
+            "Arguments": {
+                "All": "-listen:* -port:4433 -peer_uni:1 -connections:10",
+                "Loopback": "-selfsign:1",
+                "Remote": "-thumbprint:$Thumbprint -machine_cert:1 -cert_store:My"
+            }
+        },
+        "Local" : {
+            "Platform": "Windows",
+            "Tls": ["stub", "schannel", "mitls"],
+            "Arch": ["x64", "x86", "arm", "arm64"],
+            "Exe": "quicping",
+            "Arguments": {
+                "All": "-target:$RemoteAddress -port:4433 -sendbuf:0 -bind:$LocalAddress:4434 -ip:4 -core:0 -uni:1 -length:2000000000",
+                "Loopback": "",
+                "Remote": ""
+            }
+        },
+        "Iterations": 10,
+        "RemoteReadyMatcher": "Ready For Connections!",
+        "ResultsMatcher": "Closed.*\\(TX.*bytes @ (.*) kbps \\|"
+    },
+    {
+        "TestName": "Throughput",
+        "Remote" : {
+            "Platform": "Linux",
+            "Tls": ["stub", "openssl"],
+            "Arch": ["x64", "arm"],
+            "Exe": "quicping",
+            "Arguments": {
+                "All": "-listen:* -port:4433 -selfsign:1 -peer_uni:1 -connections:10",
+                "Loopback": "",
+                "Remote": ""
+            }
+        },
+        "Local" : {
+            "Platform": "linux",
+            "Tls": ["stub", "openssl"],
+            "Arch": ["x64", "arm"],
+            "Exe": "quicping",
+            "Arguments": {
+                "All": "-target:$RemoteAddress -port:4433 -sendbuf:0 -uni:1 -length:2000000000",
+                "Loopback": "",
+                "Remote": ""
+            }
+        },
+        "Iterations": 10,
+        "RemoteReadyMatcher": "Ready For Connections!",
+        "ResultsMatcher": "Closed.*\\(TX.*bytes @ (.*) kbps \\|"
+    }
+]