Backed out changeset 14e64e208672 (bug 1590167) for causing failures …

…in spurious-hyphenation-after-explicit.html and nsHyphenator.cpp

.cargo/config.in

@@ -7,11 +7,6 @@ branch = "master"
 git = "https://github.com/mozilla/neqo"
 replace-with = "vendored-sources"
 
-[source."https://github.com/jfkthame/mapped_hyph.git"]
-branch = "master"
-git = "https://github.com/jfkthame/mapped_hyph.git"
-replace-with = "vendored-sources"
-
 [source."https://github.com/hsivonen/packed_simd"]
 branch = "rust_1_32"
 git = "https://github.com/hsivonen/packed_simd"

intl/hyphenation/glue/hnjalloc.h

@@ -0,0 +1,46 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * To enable us to load hyphenation dictionaries from arbitrary resource URIs,
+ * not just through file paths using stdio, we override the (few) stdio APIs
+ * that hyphen.c uses and provide our own reimplementation that calls Gecko
+ * i/o methods.
+ */
+
+#include <stdio.h> /* ensure stdio.h is loaded before our macros */
+
+#undef FILE
+#define FILE hnjFile
+
+#define fopen(path, mode) hnjFopen(path, mode)
+#define fclose(file) hnjFclose(file)
+#define fgets(buf, count, file) hnjFgets(buf, count, file)
+#define feof(file) hnjFeof(file)
+#define fgetc(file) hnjFgetc(file)
+
+typedef struct hnjFile_ hnjFile;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void* hnj_malloc(size_t size);
+void* hnj_realloc(void* ptr, size_t size);
+void hnj_free(void* ptr);
+
+hnjFile* hnjFopen(const char* aURISpec, const char* aMode);
+
+int hnjFclose(hnjFile* f);
+
+char* hnjFgets(char* s, int n, hnjFile* f);
+
+int hnjFeof(hnjFile* f);
+
+int hnjFgetc(hnjFile* f);
+
+#ifdef __cplusplus
+}
+#endif

intl/hyphenation/glue/hnjstdio.cpp

@@ -0,0 +1,133 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file provides substitutes for the basic stdio routines used by hyphen.c
+// to read its dictionary files. We #define the stdio names to these versions
+// in hnjalloc.h, so that we can use nsIURI and nsIInputStream to specify and
+// access the dictionary resources.
+
+#include "hnjalloc.h"
+
+#undef FILE  // Undo #defines from hnjalloc.h before #including other headers
+#undef fopen
+#undef fclose
+#undef fgets
+#undef feof
+#undef fgetc
+
+#include "nsNetUtil.h"
+#include "nsIInputStream.h"
+#include "nsIURI.h"
+#include "nsContentUtils.h"
+
+#define BUFSIZE 1024
+
+struct hnjFile_ {
+  nsCOMPtr<nsIInputStream> mStream;
+  char mBuffer[BUFSIZE];
+  uint32_t mCurPos;
+  uint32_t mLimit;
+  bool mEOF;
+};
+
+// replacement for fopen()
+// (not a full substitute: only supports read access)
+hnjFile* hnjFopen(const char* aURISpec, const char* aMode) {
+  // this override only needs to support "r"
+  NS_ASSERTION(!strcmp(aMode, "r"), "unsupported fopen() mode in hnjFopen");
+
+  nsCOMPtr<nsIURI> uri;
+  nsresult rv = NS_NewURI(getter_AddRefs(uri), aURISpec);
+  if (NS_FAILED(rv)) {
+    return nullptr;
+  }
+
+  nsCOMPtr<nsIChannel> channel;
+  rv = NS_NewChannel(getter_AddRefs(channel), uri,
+                     nsContentUtils::GetSystemPrincipal(),
+                     nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL,
+                     nsIContentPolicy::TYPE_OTHER);
+  if (NS_FAILED(rv)) {
+    return nullptr;
+  }
+
+  nsCOMPtr<nsIInputStream> instream;
+  rv = channel->Open(getter_AddRefs(instream));
+  if (NS_FAILED(rv)) {
+    return nullptr;
+  }
+
+  hnjFile* f = new hnjFile;
+  f->mStream = instream;
+  f->mCurPos = 0;
+  f->mLimit = 0;
+  f->mEOF = false;
+
+  return f;
+}
+
+// replacement for fclose()
+int hnjFclose(hnjFile* f) {
+  NS_ASSERTION(f && f->mStream, "bad argument to hnjFclose");
+
+  int result = 0;
+  nsresult rv = f->mStream->Close();
+  if (NS_FAILED(rv)) {
+    result = EOF;
+  }
+  f->mStream = nullptr;
+
+  delete f;
+  return result;
+}
+
+// replacement for fgetc()
+int hnjFgetc(hnjFile* f) {
+  if (f->mCurPos >= f->mLimit) {
+    f->mCurPos = 0;
+
+    nsresult rv = f->mStream->Read(f->mBuffer, BUFSIZE, &f->mLimit);
+    if (NS_FAILED(rv)) {
+      f->mLimit = 0;
+    }
+
+    if (f->mLimit == 0) {
+      f->mEOF = true;
+      return EOF;
+    }
+  }
+
+  return f->mBuffer[f->mCurPos++];
+}
+
+// replacement for fgets()
+// (not a full reimplementation, but sufficient for libhyphen's needs)
+char* hnjFgets(char* s, int n, hnjFile* f) {
+  NS_ASSERTION(s && f, "bad argument to hnjFgets");
+
+  int i = 0;
+  while (i < n - 1) {
+    int c = hnjFgetc(f);
+
+    if (c == EOF) {
+      break;
+    }
+
+    s[i++] = c;
+
+    if (c == '\n' || c == '\r') {
+      break;
+    }
+  }
+
+  if (i == 0) {
+    return nullptr;  // end of file
+  }
+
+  s[i] = '\0';  // null-terminate the returned string
+  return s;
+}
+
+int hnjFeof(hnjFile* f) { return f->mEOF ? EOF : 0; }

intl/hyphenation/glue/moz.build

@@ -14,17 +14,16 @@ UNIFIED_SOURCES += [
     'nsHyphenator.cpp',
 ]
 
+# These files cannot be built in unified mode because they include hnjalloc.h.
+SOURCES += [
+    'hnjstdio.cpp',
+]
+
+LOCAL_INCLUDES += [
+    '../hyphen',
+]
+
 FINAL_LIBRARY = 'xul'
 
 if CONFIG['CC_TYPE'] in ('clang', 'gcc'):
     CXXFLAGS += ['-Wno-error=shadow']
-
-GENERATED_FILES += [
-    'mapped_hyph.h'
-]
-
-generated = GENERATED_FILES['mapped_hyph.h']
-generated.script = '/layout/style/RunCbindgen.py:generate'
-generated.inputs = [
-    '/third_party/rust/mapped_hyph'
-]

intl/hyphenation/glue/nsHyphenationManager.cpp

@@ -37,7 +37,8 @@ static const char kMemoryPressureNotification[] = "memory-pressure";
 static const char kParentShuttingDownNotification[] = "profile-before-change";
 static const char kChildShuttingDownNotification[] = "content-child-shutdown";
 
-class HyphenReporter final : public nsIMemoryReporter {
+class HyphenReporter final : public nsIMemoryReporter,
+                             public CountingAllocatorBase<HyphenReporter> {
  private:
   ~HyphenReporter() = default;
 
@@ -46,19 +47,14 @@ class HyphenReporter final : public nsIMemoryReporter {
 
   // For telemetry, we report the memory rounded up to the nearest KB.
   static uint32_t MemoryAllocatedInKB() {
-    size_t total = 0;
-    if (nsHyphenationManager::Instance()) {
-      total = nsHyphenationManager::Instance()->SizeOfIncludingThis(
-          moz_malloc_size_of);
-    }
-    return (total + 1023) / 1024;
+    return (MemoryAllocated() + 1023) / 1024;
   }
 
   NS_IMETHOD CollectReports(nsIHandleReportCallback* aHandleReport,
                             nsISupports* aData, bool aAnonymize) override {
-    size_t total = 0;
+    size_t total = MemoryAllocated();
     if (nsHyphenationManager::Instance()) {
-      total = nsHyphenationManager::Instance()->SizeOfIncludingThis(
+      total += nsHyphenationManager::Instance()->SizeOfIncludingThis(
           moz_malloc_size_of);
     }
     MOZ_COLLECT_REPORT("explicit/hyphenation", KIND_HEAP, UNITS_BYTES, total,
@@ -69,6 +65,30 @@ class HyphenReporter final : public nsIMemoryReporter {
 
 NS_IMPL_ISUPPORTS(HyphenReporter, nsIMemoryReporter)
 
+template <>
+CountingAllocatorBase<HyphenReporter>::AmountType
+    CountingAllocatorBase<HyphenReporter>::sAmount(0);
+
+/**
+ * Allocation wrappers to track the amount of memory allocated by libhyphen.
+ * Note that libhyphen assumes its malloc/realloc functions are infallible!
+ */
+extern "C" {
+void* hnj_malloc(size_t aSize);
+void* hnj_realloc(void* aPtr, size_t aSize);
+void hnj_free(void* aPtr);
+};
+
+void* hnj_malloc(size_t aSize) {
+  return HyphenReporter::InfallibleCountingMalloc(aSize);
+}
+
+void* hnj_realloc(void* aPtr, size_t aSize) {
+  return HyphenReporter::InfallibleCountingRealloc(aPtr, aSize);
+}
+
+void hnj_free(void* aPtr) { HyphenReporter::CountingFree(aPtr); }
+
 nsHyphenationManager* nsHyphenationManager::sInstance = nullptr;
 
 NS_IMPL_ISUPPORTS(nsHyphenationManager, nsIObserver)
@@ -237,7 +257,7 @@ void nsHyphenationManager::LoadPatternListFromOmnijar(Omnijar::Type aType) {
   }
 
   nsZipFind* find;
-  zip->FindInit("hyphenation/hyph_*.hyf", &find);
+  zip->FindInit("hyphenation/hyph_*.dic", &find);
   if (!find) {
     return;
   }
@@ -258,7 +278,7 @@ void nsHyphenationManager::LoadPatternListFromOmnijar(Omnijar::Type aType) {
       continue;
     }
     ToLowerCase(locale);
-    locale.SetLength(locale.Length() - 4);     // strip ".hyf"
+    locale.SetLength(locale.Length() - 4);     // strip ".dic"
     locale.Cut(0, locale.RFindChar('/') + 1);  // strip directory
     if (StringBeginsWith(locale, NS_LITERAL_CSTRING("hyph_"))) {
       locale.Cut(0, 5);
@@ -303,13 +323,13 @@ void nsHyphenationManager::LoadPatternListFromDir(nsIFile* aDir) {
     file->GetLeafName(dictName);
     NS_ConvertUTF16toUTF8 locale(dictName);
     ToLowerCase(locale);
-    if (!StringEndsWith(locale, NS_LITERAL_CSTRING(".hyf"))) {
+    if (!StringEndsWith(locale, NS_LITERAL_CSTRING(".dic"))) {
       continue;
     }
     if (StringBeginsWith(locale, NS_LITERAL_CSTRING("hyph_"))) {
       locale.Cut(0, 5);
     }
-    locale.SetLength(locale.Length() - 4);  // strip ".hyf"
+    locale.SetLength(locale.Length() - 4);  // strip ".dic"
     for (uint32_t i = 0; i < locale.Length(); ++i) {
       if (locale[i] == '_') {
         locale.Replace(i, 1, '-');
@@ -363,6 +383,9 @@ size_t nsHyphenationManager::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) {
   // finds it is worthwhile.
 
   result += mHyphenators.ShallowSizeOfExcludingThis(aMallocSizeOf);
+  for (auto i = mHyphenators.ConstIter(); !i.Done(); i.Next()) {
+    result += aMallocSizeOf(i.Data().get());
+  }
 
   return result;
 }