Add Wasm testing in Chromium, not just command-line v8 (dotnet#40786)

This should give us a better idea of any cases where browser and JS engine behaviour diverge. * Uses WebSockets to write pipe console messages to the test runner. * Skip System.Net.Http FunctionalTests depending on X509 certificates, tests discovery fails on `System.Net.Http.Functional.Tests.HttpClientEKUTest..cctor()` which ends up throwing PNSE * Added environment variable to help platform detect when we're running in the browser vs v8 * System.Net.Http LoopbackServer: fix NullReferenceException because we were trying to dispose a Socket instance that threw PNSE in the constructor Co-authored-by: Ankit Jain <[email protected]> Co-authored-by: Steve Pfister <[email protected]> Co-authored-by: Larry Ewing <[email protected]> Co-authored-by: Alexander Köplinger <[email protected]>
JimBobSquarePants · Sep 29, 2020 · b25c7c4 · b25c7c4
1 parent 30bf648
commit b25c7c4
Show file tree

Hide file tree

Showing 13 changed files with 123 additions and 15 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -15,7 +15,7 @@
       ]
     },
     "microsoft.dotnet.xharness.cli": {
-      "version": "1.0.0-prerelease.20403.2",
+      "version": "1.0.0-prerelease.20474.1",
       "commands": [
         "xharness"
       ]

diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml
@@ -300,6 +300,49 @@ jobs:
           eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
           eq(variables['isFullMatrix'], true))
 
+#
+# Build the whole product using Mono and run libraries tests, multi-scenario
+#
+- template: /eng/pipelines/common/platform-matrix.yml
+  parameters:
+    jobTemplate: /eng/pipelines/common/global-build-job.yml
+    helixQueuesTemplate: /eng/pipelines/libraries/helix-queues-setup.yml
+    buildConfig: Release
+    runtimeFlavor: mono
+    platforms:
+    - Browser_wasm
+    variables:
+      # map dependencies variables to local variables
+      - name: librariesContainsChange
+        value: $[ dependencies.checkout.outputs['SetPathVars_libraries.containsChange'] ]
+      - name: monoContainsChange
+        value: $[ dependencies.checkout.outputs['SetPathVars_mono.containsChange'] ]
+    jobParameters:
+      testGroup: innerloop
+      nameSuffix: AllSubsets_Mono
+      buildArgs: -s mono+libs+installer+libs.tests -c $(_BuildConfig) /p:ArchiveTests=true
+      timeoutInMinutes: 120
+      condition: >-
+        or(
+          eq(dependencies.checkout.outputs['SetPathVars_libraries.containsChange'], true),
+          eq(dependencies.checkout.outputs['SetPathVars_mono.containsChange'], true),
+          eq(dependencies.checkout.outputs['SetPathVars_installer.containsChange'], true),
+          eq(variables['isFullMatrix'], true))
+      # extra steps, run tests
+      extraStepsTemplate: /eng/pipelines/libraries/helix.yml
+      extraStepsParameters:
+        creator: dotnet-bot
+        testRunNamePrefixSuffix: Mono_$(_BuildConfig)
+        scenarios:
+        - normal
+        - wasmtestonbrowser
+        condition: >-
+          or(
+          eq(variables['librariesContainsChange'], true),
+          eq(variables['monoContainsChange'], true),
+          eq(variables['isFullMatrix'], true))
+
+
 #
 # Build the whole product using Mono and run libraries tests
 #
@@ -311,7 +354,6 @@ jobs:
     runtimeFlavor: mono
     platforms:
     - iOS_x64
-    - Browser_wasm
     variables:
       # map dependencies variables to local variables
       - name: librariesContainsChange

diff --git a/eng/testing/WasmRunnerTemplate.sh b/eng/testing/WasmRunnerTemplate.sh
@@ -14,6 +14,10 @@ else
 	HARNESS_RUNNER="dotnet xharness"
 fi
 
+if [ -z "$XHARNESS_COMMAND" ]; then
+	XHARNESS_COMMAND="test"
+fi
+
 # RunCommands defined in tests.mobile.targets
 [[RunCommands]]
 

diff --git a/eng/testing/tests.mobile.targets b/eng/testing/tests.mobile.targets
@@ -14,7 +14,7 @@
 
   <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
     <!-- We need to set this in order to get extensibility on xunit category traits and other arguments we pass down to xunit via MSBuild properties -->
-    <RunScriptCommand>$HARNESS_RUNNER wasm test --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT --  $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
+    <RunScriptCommand>$HARNESS_RUNNER wasm $XHARNESS_COMMAND  --app=. --engine=$(JSEngine) $(JSEngineArgs) --js-file=runtime.js -v --output-directory=$XHARNESS_OUT --  $(RunTestsJSArguments) --run WasmTestRunner.dll $(AssemblyName).dll</RunScriptCommand>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.cs b/src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.cs
@@ -2725,7 +2725,7 @@ await LoopbackServerFactory.CreateServerAsync(async (server, rootUrl) =>
         }
 #endregion
 
-        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsSubsystemForLinux))] // [ActiveIssue("https://github.com/dotnet/runtime/issues/18258")]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsNotWindowsSubsystemForLinux), nameof(PlatformDetection.IsNotBrowserDomSupported))] // [ActiveIssue("https://github.com/dotnet/runtime/issues/18258")]
         public async Task GetAsync_InvalidUrl_ExpectedExceptionThrown()
         {
             string invalidUri = $"http://{Configuration.Sockets.InvalidHost}";

diff --git a/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs b/src/libraries/Common/tests/System/Net/Http/LoopbackServer.cs
@@ -48,7 +48,8 @@ public LoopbackServer(Options options = null)
             }
             catch
             {
-                _listenSocket.Dispose();
+                _listenSocket?.Dispose();
+                throw;
             }
         }
 

diff --git a/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs b/src/libraries/Common/tests/TestUtilities/System/PlatformDetection.cs
@@ -48,6 +48,9 @@ public static partial class PlatformDetection
         public static bool IsThreadingSupported => !IsBrowser;
         public static bool IsBinaryFormatterSupported => !IsBrowser;
 
+        public static bool IsBrowserDomSupported => GetIsBrowserDomSupported();
+        public static bool IsNotBrowserDomSupported => !IsBrowserDomSupported;
+
         // Please make sure that you have the libgdiplus dependency installed.
         // For details, see https://docs.microsoft.com/dotnet/core/install/dependencies?pivots=os-macos&tabs=netcore31#libgdiplus
         public static bool IsDrawingSupported
@@ -367,5 +370,14 @@ private static bool GetIsRunningOnMonoInterpreter()
             var val = Environment.GetEnvironmentVariable("MONO_ENV_OPTIONS");
             return (val != null && val.Contains("--interpreter"));
         }
+
+        private static bool GetIsBrowserDomSupported()
+        {
+            if (!IsBrowser)
+                return false;
+
+            var val = Environment.GetEnvironmentVariable("IsBrowserDomSupported");
+            return (val != null && val == "true");
+        }
     }
 }
diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs b/src/libraries/System.Net.Http/tests/FunctionalTests/SocketsHttpHandlerTest.cs
@@ -126,10 +126,12 @@ public sealed class SocketsHttpHandler_HttpClient_SelectedSites_Test : HttpClien
         public SocketsHttpHandler_HttpClient_SelectedSites_Test(ITestOutputHelper output) : base(output) { }
     }
 
+#if !TARGETS_BROWSER
     public sealed class SocketsHttpHandler_HttpClientEKUTest : HttpClientEKUTest
     {
         public SocketsHttpHandler_HttpClientEKUTest(ITestOutputHelper output) : base(output) { }
     }
+#endif
 
     public sealed class SocketsHttpHandler_HttpClientHandler_Decompression_Tests : HttpClientHandler_Decompression_Test
     {
@@ -941,6 +943,7 @@ public sealed class SocketsHttpHandler_SchSendAuxRecordHttpTest : SchSendAuxReco
         public SocketsHttpHandler_SchSendAuxRecordHttpTest(ITestOutputHelper output) : base(output) { }
     }
 
+    [SkipOnMono("Tests hang with chrome. To be investigated", TestPlatforms.Browser)]
     public sealed class SocketsHttpHandler_HttpClientHandlerTest : HttpClientHandlerTest
     {
         public SocketsHttpHandler_HttpClientHandlerTest(ITestOutputHelper output) : base(output) { }

diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj b/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj
@@ -2,6 +2,7 @@
   <PropertyGroup>
     <StringResourcesPath>../../src/Resources/Strings.resx</StringResourcesPath>
     <DefineConstants Condition="'$(TargetsWindows)'=='true'">$(DefineConstants);TargetsWindows</DefineConstants>
+    <DefineConstants Condition="'$(TargetsBrowser)'=='true'">$(DefineConstants);TARGETS_BROWSER</DefineConstants>
     <DefineConstants>$(DefineConstants);SYSNETHTTP_NO_OPENSSL;HTTP3</DefineConstants>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <IncludeRemoteExecutor>true</IncludeRemoteExecutor>
@@ -193,7 +194,7 @@
     <Compile Include="HttpClientHandlerTestBase.SocketsHttpHandler.cs" />
     <Compile Include="DiagnosticsTests.cs" />
     <Compile Include="HttpClientTest.cs" />
-    <Compile Include="$(CommonTestPath)System\Net\Http\HttpClientEKUTest.cs"
+    <Compile Include="$(CommonTestPath)System\Net\Http\HttpClientEKUTest.cs" Condition="'$(TargetsBrowser)' != 'true'"
              Link="Common\System\Net\Http\HttpClientEKUTest.cs" />
     <Compile Include="$(CommonTestPath)System\Net\Http\HttpClient.SelectedSitesTest.cs"
              Link="Common\System\Net\Http\HttpClient.SelectedSitesTest.cs" />

diff --git a/src/libraries/System.Private.Xml/tests/XmlReader/Tests/AsyncReaderLateInitTests.cs b/src/libraries/System.Private.Xml/tests/XmlReader/Tests/AsyncReaderLateInitTests.cs
@@ -68,7 +68,7 @@ public static void ReadAfterInitializationWithTextReaderOnAsyncReaderDoesNotThro
             }
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
         public static void ReadAsyncAfterInitializationWithUriThrows()
         {
             using (XmlReader reader = XmlReader.Create("http://test.test/test.html", new XmlReaderSettings() { Async = true }))
@@ -77,7 +77,7 @@ public static void ReadAsyncAfterInitializationWithUriThrows()
             }
         }
 
-        [Fact]
+        [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))]
         public static void ReadAfterInitializationWithUriOnAsyncReaderTrows()
         {
             using (XmlReader reader = XmlReader.Create("http://test.test/test.html", new XmlReaderSettings() { Async = true }))

diff --git a/src/libraries/sendtohelix.proj b/src/libraries/sendtohelix.proj
@@ -116,7 +116,7 @@
     <Error Condition="!Exists('$(TestEnvFilePath)')" Text="File $(TestEnvFilePath) not found!" />
   </Target>
 
-  <Target Condition="'$(Scenarios)' != ''" Name="CreateAllScenarioTestEnvFiles">
+  <Target Condition="'$(Scenarios)' != '' and '$(TargetOS)' != 'Browser'" Name="CreateAllScenarioTestEnvFiles">
     <!-- This target creates one __TestEnv file for each of the scenarios in the $(Scenarios) comma-separated list. -->
 
     <Message Importance="High" Text="Creating per-scenario TestEnv files for scenarios $(Scenarios)" />

diff --git a/src/libraries/sendtohelixhelp.proj b/src/libraries/sendtohelixhelp.proj
@@ -95,6 +95,16 @@
     <HelixPreCommand Condition="'$(TargetsWindows)' != 'true'" Include="export MONO_ENV_OPTIONS='$(MonoEnvOptions)'" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(TargetOS)' == 'Browser'">
+    <HelixPreCommand Condition="'$(Scenario)' != 'WasmTestOnBrowser'" Include="export XHARNESS_COMMAND=test" />
+    <HelixPreCommand Condition="'$(Scenario)' == 'WasmTestOnBrowser'" Include="export XHARNESS_COMMAND=test-browser" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(Scenario)' == 'WasmTestOnBrowser'">
+    <HelixPreCommand Include="export PATH=$HELIX_CORRELATION_PAYLOAD/chromedriver_linux64:$PATH" />
+    <HelixPreCommand Include="export PATH=$HELIX_CORRELATION_PAYLOAD/chrome-linux:$PATH" />
+  </ItemGroup>
+
   <!-- HelixPreCommands is a set of commands run before the work item command. We use it here to inject
        setting up the per-scenario environment.
   -->
@@ -104,8 +114,8 @@
          (which is when this project is invoked to call the "CreateTestEnvFile" target).
     -->
     <TestEnvFileName></TestEnvFileName>
-    <TestEnvFileName Condition=" '$(Scenario)' != '' and '$(TargetsWindows)' == 'true' ">SetStressModes_$(Scenario).cmd</TestEnvFileName>
-    <TestEnvFileName Condition=" '$(Scenario)' != '' and '$(TargetsWindows)' != 'true' ">SetStressModes_$(Scenario).sh</TestEnvFileName>
+    <TestEnvFileName Condition=" '$(Scenario)' != '' and '$(TargetsWindows)' == 'true'">SetStressModes_$(Scenario).cmd</TestEnvFileName>
+    <TestEnvFileName Condition=" '$(Scenario)' != '' and '$(TargetsWindows)' != 'true' and '$(TargetOS)' != 'Browser'">SetStressModes_$(Scenario).sh</TestEnvFileName>
   </PropertyGroup>
 
   <ItemGroup Condition=" '$(TestEnvFileName)' != '' and '$(TargetsWindows)' == 'true' ">
@@ -176,9 +186,19 @@
       </XHarnessAppBundleToTest>
     </ItemGroup>
 
+    <PropertyGroup Condition="'$(TargetOS)' == 'Browser'">
+      <!-- Version number to revision number mapping from http://omahaproxy.appspot.com/ -->
+      <!-- 84.0.4147.105 is 768962 for example -->
+      <ChromiumRevision>768968</ChromiumRevision>
+      <ChromiumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chrome-linux.zip</ChromiumUrl>
+      <SeleniumUrl>https://storage.googleapis.com/chromium-browser-snapshots/Linux_x64/$(ChromiumRevision)/chromedriver_linux64.zip</SeleniumUrl>
+    </PropertyGroup>
+
     <ItemGroup Condition="'$(TargetOS)' != 'iOS' and '$(TargetOS)' != 'tvOS'">
       <HelixCorrelationPayload Include="$(HelixCorrelationPayload)"
-                               Condition="'$(IncludeHelixCorrelationPayload)' == 'true'" />
+                               Condition="'$(IncludeHelixCorrelationPayload)' == 'true' and '$(TargetOS)' != 'Browser'" />
+      <HelixCorrelationPayload Include="chromium" Uri="$(ChromiumUrl)" Condition="'$(TargetOS)' == 'Browser'" />
+      <HelixCorrelationPayload Include="chromedriver" Uri="$(SeleniumUrl)" Condition="'$(TargetOS)' == 'Browser'" />
 
       <_WorkItem Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" />
 

diff --git a/src/mono/wasm/runtime-test.js b/src/mono/wasm/runtime-test.js
@@ -5,9 +5,34 @@
 
 //glue code to deal with the differences between chrome, ch, d8, jsc and sm.
 var is_browser = typeof window != "undefined";
+var consoleWebSocket;
+var print;
 
 if (is_browser) {
 	// We expect to be run by tests/runtime/run.js which passes in the arguments using http parameters
+	window.real_print = console.log;
+	print = function(_msg) { window.real_print(_msg); };
+	console.log = print;
+	console.debug = print;
+	console.error = print;
+	console.trace = print;
+	console.warn = print;
+	console.info = print;
+
+	const consoleUrl = `${window.location.origin}/console`.replace('http://', 'ws://');
+
+	consoleWebSocket = new WebSocket(consoleUrl);
+	consoleWebSocket.onopen = function(event) {
+		consoleWebSocket.send("browser: Console websocket connected.");
+
+		window.real_print = function(msg) {
+			consoleWebSocket.send(msg);
+		};
+	};
+	consoleWebSocket.onerror = function(event) {
+		console.log(`websocket error: ${event}`);
+	};
+
 	var url = new URL (decodeURI (window.location));
 	arguments = [];
 	for (var v of url.searchParams) {
@@ -18,9 +43,6 @@ if (is_browser) {
 	}
 }
 
-if (is_browser || typeof print === "undefined")
-	print = console.log;
-
 // JavaScript core does not have a console defined
 if (typeof console === "undefined") {
 	var Console = function () {
@@ -151,6 +173,9 @@ while (args !== undefined && args.length > 0) {
 }
 testArguments = args;
 
+// cheap way to let the testing infrastructure know we're running in a browser context (or not)
+setenv["IsBrowserDomSupported"] = is_browser.toString().toLowerCase();
+
 function writeContentToFile(content, path)
 {
 	var stream = FS.open(path, 'w+');
-Original file line number
+Diff line change
@@ Expand Up / @@ -48,7 +48,8 @@ public LoopbackServer(Options options = null) @@
                 }
                 catch
                 {
-                    _listenSocket.Dispose();
+                    _listenSocket?.Dispose();
+                    throw;
                 }
             }
@@ Expand Down @@