Bug 1736248 - Update the charset source if non-ASCII is seen after th…

…e first detector guess but the encoding does not change. r=smaug

Differential Revision: https://phabricator.services.mozilla.com/D133731

docshell/test/browser/browser.ini

@@ -71,6 +71,7 @@ support-files =
   file_bug1716290-2.sjs
   file_bug1716290-3.sjs
   file_bug1716290-4.sjs
+  file_bug1736248-1.html
 
 [browser_TopLevelNavigationDelegate.js]
 support-files =
@@ -231,3 +232,4 @@ https_first_disabled = true
 [browser_bfcache_copycommand.js]
 skip-if =
   os == "linux" && bits == 64 # Bug 1730593
+[browser_bug1736248-1.js]

docshell/test/browser/browser_bug1736248-1.js

@@ -0,0 +1,34 @@
+function test() {
+  var rootDir = "http://mochi.test:8888/browser/docshell/test/browser/";
+  runCharsetTest(
+    rootDir + "file_bug1736248-1.html",
+    afterOpen,
+    afterChangeCharset
+  );
+}
+
+function afterOpen() {
+  is(
+    content.document.documentElement.textContent.indexOf("\u00C3"),
+    1064,
+    "Doc should be windows-1252 initially"
+  );
+  is(
+    content.document.characterSet,
+    "windows-1252",
+    "Doc should report windows-1252 initially"
+  );
+}
+
+function afterChangeCharset() {
+  is(
+    content.document.documentElement.textContent.indexOf("\u00E4"),
+    1064,
+    "Doc should be UTF-8 subsequently"
+  );
+  is(
+    content.document.characterSet,
+    "UTF-8",
+    "Doc should report UTF-8 subsequently"
+  );
+}

docshell/test/browser/file_bug1736248-1.html

@@ -0,0 +1,4 @@
+Kilobyte of ASCII followed by UTF-8.
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+
+Hej världen!

parser/html/nsHtml5StreamParser.cpp

@@ -2273,6 +2273,11 @@ void nsHtml5StreamParser::ParseAvailableData() {
                     source == kCharsetFromFinalUserForcedAutoDetection);
                 mTreeBuilder->NeedsCharsetSwitchTo(encoding, source, 0);
                 requestedReload = true;
+              } else if (mCharsetSource ==
+                             kCharsetFromInitialAutoDetectionASCII &&
+                         mDetectorHasSeenNonAscii) {
+                mCharsetSource = source;
+                mTreeBuilder->SetDocumentCharset(mEncoding, mCharsetSource);
               }
             }