Run wasm tests on Helix (dotnet#37752)

Build Wasm for testing via Helix. A bunch of test suites are disabled on Wasm here, and will be re-enabled as we get the up and running.
godspeedu · Jun 25, 2020 · a926345 · a926345
1 parent 5c2d025
commit a926345
Show file tree

Hide file tree

Showing 17 changed files with 207 additions and 41 deletions.
diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml
@@ -178,5 +178,9 @@
       <Uri>https://github.com/dotnet/xharness</Uri>
       <Sha>19aa97963aa11b4818f3e3487b913244efa4ff0f</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.DotNet.XHarness.CLI" Version="1.0.0-prerelease.20322.1">
+      <Uri>https://github.com/dotnet/xharness</Uri>
+      <Sha>19aa97963aa11b4818f3e3487b913244efa4ff0f</Sha>
+    </Dependency>
   </ToolsetDependencies>
 </Dependencies>
diff --git a/eng/Versions.props b/eng/Versions.props
@@ -106,6 +106,7 @@
     <!-- Testing -->
     <MicrosoftNETTestSdkVersion>16.7.0-release-20200612-02</MicrosoftNETTestSdkVersion>
     <MicrosoftDotNetXHarnessTestRunnersXunitVersion>1.0.0-prerelease.20322.1</MicrosoftDotNetXHarnessTestRunnersXunitVersion>
+    <MicrosoftDotNetXHarnessCLIVersion>1.0.0-prerelease.20322.1</MicrosoftDotNetXHarnessCLIVersion>
     <XUnitVersion>2.4.1</XUnitVersion>
     <XUnitRunnerVisualStudioVersion>2.4.2</XUnitRunnerVisualStudioVersion>
     <CoverletCollectorVersion>1.3.0</CoverletCollectorVersion>

diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml
@@ -12,6 +12,8 @@ parameters:
   pool: ''
   condition: true
   isOfficialBuild: false
+  runtimeFlavor: 'coreclr'
+  helixQueues: ''
   extraStepsTemplate: ''
   extraStepsParameters: {}
 
@@ -81,6 +83,8 @@ jobs:
           osSubgroup: ${{ parameters.osSubgroup }}
           archType: ${{ parameters.archType }}
           buildConfig: ${{ parameters.buildConfig }}
+          runtimeFlavor: ${{ parameters.runtimeFlavor }}
+          helixQueues: ${{ parameters.helixQueues }}
           ${{ insert }}: ${{ parameters.extraStepsParameters }}
 
     - task: PublishBuildArtifacts@1

diff --git a/eng/pipelines/libraries/build-job.yml b/eng/pipelines/libraries/build-job.yml
@@ -174,14 +174,11 @@ jobs:
         - ${{ if eq(parameters.runTests, true) }}:
           - template: /eng/pipelines/libraries/helix.yml
             parameters:
-              targetOS: ${{ parameters.osGroup }}
-              archGroup: ${{ parameters.archType }}
-              configuration: ${{ parameters.buildConfig }}
-              helixQueues: ${{ join('+', parameters.helixQueues) }}
-              msbuildScript: $(_msbuildCommand)
-              framework: ${{ parameters.framework }}
+              osGroup: ${{ parameters.osGroup }}
+              archType: ${{ parameters.archType }}
+              buildConfig: ${{ parameters.buildConfig }}
+              helixQueues: ${{ parameters.helixQueues }}
               testScope: ${{ parameters.testScope }}
               creator: dotnet-bot
-              helixToken: ''
               testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix)
               extraHelixArguments: $(_extraHelixArguments)
diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml
@@ -144,4 +144,8 @@ jobs:
     - ${{ if eq(parameters.platform, 'Windows_NT_arm64') }}:
       - Windows.10.Arm64.Open
 
+    # WebAssembly
+    - ${{ if eq(parameters.platform, 'Browser_wasm') }}:
+      - Ubuntu.2004.Amd64.Open
+
     ${{ insert }}: ${{ parameters.jobParameters }}
diff --git a/eng/pipelines/libraries/helix.yml b/eng/pipelines/libraries/helix.yml
@@ -1,12 +1,10 @@
 parameters:
   runtimeFlavor: ''
-  archGroup: ''
-  configuration: ''
+  archType: ''
+  buildConfig: ''
   creator: ''
-  framework: ''
   helixQueues: ''
-  msbuildScript: ''
-  targetOS: ''
+  osGroup: ''
   testRunNamePrefixSuffix: ''
   testScope: 'innerloop' # innerloop | outerloop | all
   interpreter: ''
@@ -15,12 +13,12 @@ parameters:
   scenarios: ''
 
 steps:
-  - script: ${{ parameters.msbuildScript }}
+  - script: $(_msbuildCommand)
             $(Build.SourcesDirectory)/src/libraries/sendtohelix.proj
             /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
-            /p:TargetArchitecture=${{ parameters.archGroup }}
-            /p:Configuration=${{ parameters.configuration }}
-            /p:TargetOS=${{ parameters.targetOS }}
+            /p:TargetArchitecture=${{ parameters.archType }}
+            /p:Configuration=${{ parameters.buildConfig }}
+            /p:TargetOS=${{ parameters.osGroup }}
             /p:MonoEnableInterpreter=${{ parameters.interpreter }}
             /p:TestScope=${{ parameters.testScope }}
             /p:TestRunNamePrefixSuffix=${{ parameters.testRunNamePrefixSuffix }}
@@ -32,5 +30,5 @@ steps:
     condition: and(succeeded(), ${{ parameters.condition }})
     env:
       SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops
-      HelixTargetQueues: ${{ parameters.helixQueues }} # Pass queues to MSBuild as env var to avoid need of escaping them
+      HelixTargetQueues: ${{ join('+', parameters.helixQueues) }} # Pass queues to MSBuild as env var to avoid need of escaping them
       _Scenarios: ${{ join(',', parameters.scenarios) }} # Pass scenarios to MSBuild as env var to avoid need of escaping comma separated list
diff --git a/eng/pipelines/libraries/run-test-job.yml b/eng/pipelines/libraries/run-test-job.yml
@@ -92,16 +92,13 @@ jobs:
         - template: /eng/pipelines/libraries/helix.yml
           parameters:
             runtimeFlavor: ${{ parameters.runtimeFlavor }}
-            targetOS: ${{ parameters.osGroup }}
-            archGroup: ${{ parameters.archType }}
-            configuration: ${{ parameters.buildConfig }}
-            helixQueues: ${{ join('+', parameters.helixQueues) }}
-            msbuildScript: $(_msbuildCommand)
-            framework: ${{ parameters.framework }}
+            osGroup: ${{ parameters.osGroup }}
+            archType: ${{ parameters.archType }}
+            buildConfig: ${{ parameters.buildConfig }}
+            helixQueues: ${{ parameters.helixQueues }}
             testScope: ${{ parameters.testScope }}
             interpreter: ${{ parameters.interpreter }}
             creator: dotnet-bot
-            helixToken: ''
             testRunNamePrefixSuffix: $(_testRunNamePrefixSuffix)
             extraHelixArguments: $(_extraHelixArguments)
 

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -247,7 +247,6 @@ jobs:
     - tvOS_x64
     - iOS_arm64
     - iOS_x86
-    - Browser_wasm
     - Linux_arm
     jobParameters:
       testGroup: innerloop
@@ -283,6 +282,33 @@ jobs:
           eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
           eq(variables['isFullMatrix'], true))
 
+#
+# Build the whole product using Mono and run tests
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+    buildConfig: Release
+    runtimeFlavor: mono
+    platforms:
+    - Browser_wasm
+    jobParameters:
+      testGroup: innerloop
+      nameSuffix: AllSubsets_Mono
+      buildArgs: -s mono+libs+installer+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true
+      timeoutInMinutes: 120
+      extraStepsTemplate: /eng/pipelines/libraries/helix.yml
+      extraStepsParameters:
+        creator: dotnet-bot
+        testRunNamePrefixSuffix: Mono_$(_BuildConfig)
+      condition: >-
+        or(
+          eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true),
+          eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
+          eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
+          eq(variables['isFullMatrix'], true))
+
 #
 # Build Mono and Installer on LLVMJIT mode
 #

diff --git a/eng/testing/WasmRunnerTemplate.sh b/eng/testing/WasmRunnerTemplate.sh
@@ -1,16 +1,17 @@
 #!/usr/bin/env bash
 
 EXECUTION_DIR=$(dirname $0)
-JAVASCRIPT_ENGINE=$1
 
 cd $EXECUTION_DIR
 
 XHARNESS_OUT="$EXECUTION_DIR/xharness-output"
 
-dotnet xharness wasm test --engine=$JAVASCRIPT_ENGINE \
-    --js-file=runtime.js -v \
-    --output-directory=$XHARNESS_OUT \
-    -- --enable-gc --run WasmTestRunner.dll ${@:2}
+if [ ! -x "$(command -v xharness)" ]; then
+	HARNESS_RUNNER="dotnet"
+fi
+
+# RunCommands defined in tests.mobile.targets
+[[RunCommands]]
 
 _exitCode=$?
 

diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets
@@ -7,6 +7,15 @@
     <JSEngine Condition="'$(TargetOS)' == 'Browser' and '$(JSEngine)' == ''">V8</JSEngine>
   </PropertyGroup>
 
+  <ItemGroup>
+    <RunScriptCommands Include="TARGET_ARCH=$(TargetArchitecture)" />
+  </ItemGroup>
+
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <!-- We need to set this in order to get extensibility on xunit category traits and other arguments we pass down to xunit via MSBuild properties -->
+    <RunScriptCommand>$HARNESS_RUNNER xharness wasm test --engine=$(JSEngine) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT -- --enable-gc --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
+  </PropertyGroup>
+
   <!-- Generate a self-contained app bundle for Android with tests. -->
   <UsingTask Condition="'$(TargetOS)' == 'Android'"
              TaskName="AndroidAppBuilderTask"
@@ -156,5 +165,5 @@
   <Target Name="PublishTestAsSelfContained"
           Condition="'$(IsCrossTargetingBuild)' != 'true'"
           AfterTargets="Build"
-          DependsOnTargets="UpdateRuntimePack;Publish;BundleTestAppleApp;BundleTestAndroidApp;BundleTestWasmApp" />
+          DependsOnTargets="UpdateRuntimePack;Publish;BundleTestAppleApp;BundleTestAndroidApp;BundleTestWasmApp;ArchiveTests" />
 </Project>
diff --git a/eng/testing/tests.targets b/eng/testing/tests.targets
@@ -24,15 +24,30 @@
     <RunScriptCommands Include="set DEVPATH=%RUNTIME_PATH%" />
   </ItemGroup>
 
+  <PropertyGroup>
+    <ArchiveTestsAfterTargets>PrepareForRun</ArchiveTestsAfterTargets>
+
+    <!-- For browser we need to hook up the target with DependsOnTargets in PublishTestAsSelfContained
+    because we do a Publish which runs after Build, if we run after PrepareForRun we would generated
+    an empty zip because we haven't published the selfcontained app.  -->
+    <ArchiveTestsAfterTargets Condition="'$(TargetOS)' == 'Browser'" />
+  </PropertyGroup>
+
   <!-- Archive test binaries. -->
   <Target Name="ArchiveTests"
-          Condition="'$(ArchiveTests)' == 'true' and '$(IgnoreForCI)' != 'true'"
-          AfterTargets="PrepareForRun"
+          Condition="'$(ArchiveTests)' == 'true' and '$(IgnoreForCI)' != 'true' and ('$(TargetsMobile)' != 'true' or '$(TargetOS)' == 'Browser')"
+          AfterTargets="$(ArchiveTestsAfterTargets)"
           DependsOnTargets="GenerateRunScript">
     <Error Condition="'$(TestArchiveTestsDir)' == ''" Text="TestArchiveTestsDir property to archive the test folder must be set." />
 
+    <PropertyGroup>
+      <BundleDir>$([MSBuild]::NormalizeDirectory('$(OutDir)', 'AppBundle'))</BundleDir>
+      <_ZipSourceDirectory>$(OutDir)</_ZipSourceDirectory>
+      <_ZipSourceDirectory Condition="'$(TargetOS)' == 'Browser'">$(BundleDir)</_ZipSourceDirectory>
+    </PropertyGroup>
+
     <MakeDir Directories="$(TestArchiveTestsDir)" />
-    <ZipDirectory SourceDirectory="$(OutDir)"
+    <ZipDirectory SourceDirectory="$(_ZipSourceDirectory)"
                   DestinationFile="$([MSBuild]::NormalizePath('$(TestArchiveTestsDir)', '$(TestProjectName).zip'))"
                   Overwrite="true" />
   </Target>

diff --git a/eng/testing/xunit/xunit.console.targets b/eng/testing/xunit/xunit.console.targets
@@ -4,7 +4,7 @@
     <TestResultsName>testResults.xml</TestResultsName>
   </PropertyGroup>
 
-  <PropertyGroup>
+  <PropertyGroup Condition="'$(TargetsMobile)' != 'true'">
     <_depsFileArgument Condition="'$(GenerateDependencyFile)' == 'true'">--depsfile $(AssemblyName).deps.json</_depsFileArgument>
     <RunScriptCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'">"$(RunScriptHost)" exec --runtimeconfig $(AssemblyName).runtimeconfig.json $(_depsFileArgument) xunit.console.dll</RunScriptCommand>
     <RunScriptCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">xunit.console.exe</RunScriptCommand>
@@ -13,13 +13,15 @@
     <RunScriptCommand>$(RunScriptCommand) -xml $(TestResultsName)</RunScriptCommand>
     <RunScriptCommand>$(RunScriptCommand) -nologo</RunScriptCommand>
     <RunScriptCommand Condition="'$(ArchiveTests)' == 'true'">$(RunScriptCommand) -nocolor</RunScriptCommand>
+    <RunScriptCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' and '$(TestDisableAppDomain)' == 'true'">$(RunScriptCommand) -noappdomain</RunScriptCommand>
+    <RunScriptCommand Condition="'$(TestDisableParallelization)' == 'true'">$(RunScriptCommand) -maxthreads 1</RunScriptCommand>
+    <RunScriptCommand Condition="'$(XUnitShowProgress)' == 'true'">$(RunScriptCommand) -verbose</RunScriptCommand>
+  </PropertyGroup>
 
+  <PropertyGroup>
     <!-- Add local and global options to the argument stack. -->
-    <RunScriptCommand Condition="'$(TestDisableParallelization)' == 'true'">$(RunScriptCommand) -maxthreads 1</RunScriptCommand>
     <RunScriptCommand Condition="'$(XUnitMethodName)' != ''">$(RunScriptCommand) -method $(XUnitMethodName)</RunScriptCommand>
     <RunScriptCommand Condition="'$(XUnitClassName)' != ''">$(RunScriptCommand) -class $(XUnitClassName)</RunScriptCommand>
-    <RunScriptCommand Condition="'$(XUnitShowProgress)' == 'true'">$(RunScriptCommand) -verbose</RunScriptCommand>
-    <RunScriptCommand Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework' and '$(TestDisableAppDomain)' == 'true'">$(RunScriptCommand) -noappdomain</RunScriptCommand>
 
     <!-- Add to run argument string -->
     <RunScriptCommand>$(RunScriptCommand)$(_withCategories.Replace(';', ' -trait category='))</RunScriptCommand>

diff --git a/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs b/src/libraries/System.Formats.Cbor/tests/Reader/CborReaderTests.cs
@@ -233,6 +233,7 @@ public static void ReadEncodedValue_InvalidConformance_ConformanceCheckDisabled_
         }
 
         [Theory]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/37669", TestPlatforms.Browser)]
         [InlineData("a501020326200121582065eda5a12577c2bae829437fe338701a10aaa375e1bb5b5de108de439c08551d2258201e52ed75701163f7f9e40ddf9f341b3dc9ba860af7e0ca7ca7e9eecd0084d19c",
                     "65eda5a12577c2bae829437fe338701a10aaa375e1bb5b5de108de439c08551d",
                     "1e52ed75701163f7f9e40ddf9f341b3dc9ba860af7e0ca7ca7e9eecd0084d19c",

diff --git a/src/libraries/System.Formats.Cbor/tests/Writer/CborWriterTests.cs b/src/libraries/System.Formats.Cbor/tests/Writer/CborWriterTests.cs
@@ -342,6 +342,7 @@ public static void InvalidConformanceMode_ShouldThrowArgumentOutOfRangeException
         public static IEnumerable<object[]> EncodedValueBadInputs => CborReaderTests.InvalidCborValues.Select(x => new[] { x });
 
         [Theory]
+        [ActiveIssue("https://github.com/dotnet/runtime/issues/37669", TestPlatforms.Browser)]
         [InlineData("a501020326200121582065eda5a12577c2bae829437fe338701a10aaa375e1bb5b5de108de439c08551d2258201e52ed75701163f7f9e40ddf9f341b3dc9ba860af7e0ca7ca7e9eecd0084d19c",
                     "65eda5a12577c2bae829437fe338701a10aaa375e1bb5b5de108de439c08551d",
                     "1e52ed75701163f7f9e40ddf9f341b3dc9ba860af7e0ca7ca7e9eecd0084d19c",

diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj
@@ -144,7 +144,7 @@
   <Target Name="CompressRuntimeDirectory"
           Inputs="@(_RuntimeInputs);@(TestArchiveRuntimeDependency)"
           Outputs="$(TestArchiveRuntimeFile)"
-          Condition="'$(BuildAllConfigurations)' != 'true'">
+          Condition="'$(BuildAllConfigurations)' != 'true' and '$(TargetsMobile)' != 'true'">
 
     <!-- Compress the test files, testhost, and per-scenario scripts into a single ZIP file for sending to the Helix machines. -->
 
@@ -153,7 +153,6 @@
     <Message Importance="High" Text="Creating directory $(TestArchiveRuntimeRoot)" />
     <MakeDir Directories="$(TestArchiveRuntimeRoot)" />
 
-    <Message Importance="High" Text="ZIP directory $(TestHostRootPath) to $(TestArchiveRuntimeFile)" />
     <ZipDirectory SourceDirectory="$(TestHostRootPath)"
                   DestinationFile="$(TestArchiveRuntimeFile)"
                   Overwrite="true" />

diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj
@@ -62,6 +62,11 @@
     <HelixType Condition="'$(BuildAllConfigurations)' == 'true'">test/functional/packaging/</HelixType>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+    <IncludeDotNetCli>true</IncludeDotNetCli>
+    <IncludeXHarnessCli>true</IncludeXHarnessCli>
+  </PropertyGroup>
+
   <ItemGroup Condition="'$(HelixCommand)' == '' and '$(BuildAllConfigurations)' == 'true'">
     <HelixPreCommand Include="set DOTNET_CLI_TELEMETRY_OPTOUT=1" />
     <HelixPreCommand Include="set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1" />
@@ -147,7 +152,7 @@
     <Message Importance="High" Text="Using Timeout: $(_timeoutSpan)" />
 
     <ItemGroup>
-      <HelixCorrelationPayload Include="$(HelixCorrelationPayload)" />
+      <HelixCorrelationPayload Condition="'$(TargetOS)' != 'Browser'" Include="$(HelixCorrelationPayload)" />
 
       <_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" />