Font subset (flutter#14828)

BUILD.gn

@@ -31,6 +31,10 @@ group("flutter") {
     "$flutter_root/sky",
   ]
 
+  if (current_toolchain == host_toolchain) {
+    public_deps += [ "$flutter_root/tools/font-subset" ]
+  }
+
   if (current_toolchain == host_toolchain) {
     public_deps += [ "$flutter_root/shell/testing" ]
   }

testing/run_tests.py

@@ -20,6 +20,7 @@
 fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
 roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf')
 dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',)
+font_subset_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'font-subset')
 
 fml_unittests_filter = '--gtest_filter=-*TimeSensitiveTest*:*GpuThreadMerger*'
 
@@ -326,7 +327,7 @@ def main():
   args = parser.parse_args()
 
   if args.type == 'all':
-    types = ['engine', 'dart', 'benchmarks', 'java']
+    types = ['engine', 'dart', 'benchmarks', 'java', 'font-subset']
   else:
     types = args.type.split(',')
 
@@ -355,6 +356,9 @@ def main():
   if 'benchmarks' in types and not IsWindows():
     RunEngineBenchmarks(build_dir, engine_filter)
 
+  if 'engine' in types or 'font-subset' in types:
+    RunCmd(['python', 'test.py'], cwd=font_subset_dir)
+
 
 if __name__ == '__main__':
   sys.exit(main())

tools/font-subset/.gitignore

@@ -0,0 +1 @@
+gen/*.ttf

tools/font-subset/BUILD.gn

@@ -0,0 +1,23 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+executable("font-subset") {
+  sources = [
+    "hb_wrappers.h",
+    "main.cc",
+  ]
+
+  deps = [
+    "//third_party/harfbuzz",
+  ]
+
+  libs = []
+  if (is_mac) {
+    libs += [
+      "Foundation.framework",
+      "CoreGraphics.framework",
+      "CoreText.framework",
+    ]
+  }
+}

tools/font-subset/hb_wrappers.h

@@ -0,0 +1,35 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#ifndef HB_WRAPPERS_H_
+#define HB_WRAPPERS_H_
+
+#include <hb-subset.h>
+
+namespace HarfbuzzWrappers {
+struct hb_blob_deleter {
+  void operator()(hb_blob_t* ptr) { hb_blob_destroy(ptr); }
+};
+
+struct hb_face_deleter {
+  void operator()(hb_face_t* ptr) { hb_face_destroy(ptr); }
+};
+
+struct hb_subset_input_deleter {
+  void operator()(hb_subset_input_t* ptr) { hb_subset_input_destroy(ptr); }
+};
+
+struct hb_set_deleter {
+  void operator()(hb_set_t* ptr) { hb_set_destroy(ptr); }
+};
+
+using HbBlobPtr = std::unique_ptr<hb_blob_t, hb_blob_deleter>;
+using HbFacePtr = std::unique_ptr<hb_face_t, hb_face_deleter>;
+using HbSubsetInputPtr =
+    std::unique_ptr<hb_subset_input_t, hb_subset_input_deleter>;
+using HbSetPtr = std::unique_ptr<hb_set_t, hb_set_deleter>;
+
+};  // namespace HarfbuzzWrappers
+
+#endif  // HB_WRAPPERS_H_s

tools/font-subset/main.cc

@@ -0,0 +1,136 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include <hb-subset.h>
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <limits>
+#include <set>
+#include <string>
+
+#include "hb_wrappers.h"
+
+hb_codepoint_t ParseCodepoint(const std::string& arg) {
+  unsigned long value = 0;
+  // Check for \u123, u123, otherwise let strtoul work it out.
+  if (arg[0] == 'u') {
+    value = strtoul(arg.c_str() + 1, nullptr, 16);
+  } else if (arg[0] == '\\' && arg[1] == 'u') {
+    value = strtoul(arg.c_str() + 2, nullptr, 16);
+  } else {
+    value = strtoul(arg.c_str(), nullptr, 0);
+  }
+  if (value == 0 || value > std::numeric_limits<hb_codepoint_t>::max()) {
+    std::cerr << "The value '" << arg << "' (" << value
+              << ") could not be parsed as a valid unicode codepoint; aborting."
+              << std::endl;
+    exit(-1);
+  }
+  return value;
+}
+
+void Usage() {
+  std::cout << "Usage:" << std::endl;
+  std::cout << "font-subset <output.ttf> <input.ttf>" << std::endl;
+  std::cout << std::endl;
+  std::cout << "The output.ttf file will be overwritten if it exists already "
+               "and the subsetting operation succeeds."
+            << std::endl;
+  std::cout << "Codepoints should be specified on stdin, separated by spaces, "
+               "and must be input as decimal numbers (123), hexidecimal "
+               "numbers (0x7B), or unicode hexidecimal characters (\\u7B)."
+            << std::endl;
+  std::cout << "Input terminates with a newline." << std::endl;
+  std::cout
+      << "This program will de-duplicate codepoints if the same codepoint is "
+         "specified multiple times, e.g. '123 123' will be treated as '123'."
+      << std::endl;
+}
+
+int main(int argc, char** argv) {
+  if (argc != 3) {
+    Usage();
+    return -1;
+  }
+  std::string output_file_path(argv[1]);
+  std::string input_file_path(argv[2]);
+  std::cout << "Using output file: " << output_file_path << std::endl;
+  std::cout << "Using source file: " << input_file_path << std::endl;
+
+  HarfbuzzWrappers::HbBlobPtr font_blob(
+      hb_blob_create_from_file(input_file_path.c_str()));
+  if (!hb_blob_get_length(font_blob.get())) {
+    std::cerr << "Failed to load input font " << input_file_path
+              << "; aborting." << std::endl;
+    return -1;
+  }
+
+  HarfbuzzWrappers::HbFacePtr font_face(hb_face_create(font_blob.get(), 0));
+  if (font_face.get() == hb_face_get_empty()) {
+    std::cerr << "Failed to load input font face " << input_file_path
+              << "; aborting." << std::endl;
+    return -1;
+  }
+
+  HarfbuzzWrappers::HbSubsetInputPtr input(hb_subset_input_create_or_fail());
+  {
+    hb_set_t* desired_codepoints = hb_subset_input_unicode_set(input.get());
+    HarfbuzzWrappers::HbSetPtr actual_codepoints(hb_set_create());
+    hb_face_collect_unicodes(font_face.get(), actual_codepoints.get());
+    std::string raw_codepoint;
+    while (std::cin >> raw_codepoint) {
+      auto codepoint = ParseCodepoint(raw_codepoint);
+      if (!codepoint) {
+        std::cerr << "Invalid codepoint for " << raw_codepoint << "; exiting."
+                  << std::endl;
+        return -1;
+      }
+      if (!hb_set_has(actual_codepoints.get(), codepoint)) {
+        std::cerr << "Codepoint " << raw_codepoint
+                  << " not found in font, aborting." << std::endl;
+        return -1;
+      }
+      hb_set_add(desired_codepoints, codepoint);
+    }
+    if (hb_set_is_empty(desired_codepoints)) {
+      std::cerr << "No codepoints specified, exiting." << std::endl;
+      return -1;
+    }
+  }
+
+  HarfbuzzWrappers::HbFacePtr new_face(hb_subset(font_face.get(), input.get()));
+
+  if (new_face.get() == hb_face_get_empty()) {
+    std::cerr << "Failed to subset font; aborting." << std::endl;
+    return -1;
+  }
+
+  HarfbuzzWrappers::HbBlobPtr result(hb_face_reference_blob(new_face.get()));
+  if (!hb_blob_get_length(result.get())) {
+    std::cerr << "Failed get new font bytes; aborting" << std::endl;
+    return -1;
+  }
+
+  unsigned int data_length;
+  const char* data = hb_blob_get_data(result.get(), &data_length);
+
+  std::ofstream output_font_file;
+  output_font_file.open(output_file_path,
+                        std::ios::out | std::ios::trunc | std::ios::binary);
+  if (!output_font_file.is_open()) {
+    std::cerr << "Failed to open output file '" << output_file_path
+              << "'. The parent directory may not exist, or the user does not "
+                 "have permission to create this file."
+              << std::endl;
+    return -1;
+  }
+  output_font_file.write(data, data_length);
+  output_font_file.flush();
+  output_font_file.close();
+
+  std::cout << "Wrote " << data_length << " bytes to " << output_file_path
+            << std::endl;
+  return 0;
+}

tools/font-subset/test.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python
+#
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+'''
+Tests for font-subset
+'''
+
+import filecmp
+import os
+import subprocess
+import sys
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+SRC_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, '..', '..', '..'))
+MATERIAL_TTF = os.path.join(SCRIPT_DIR, 'fixtures', 'MaterialIcons-Regular.ttf')
+IS_WINDOWS = sys.platform.startswith(('cygwin', 'win'))
+EXE = '.exe' if IS_WINDOWS else ''
+BAT = '.bat' if IS_WINDOWS else ''
+FONT_SUBSET = os.path.join(SRC_DIR, 'out', 'host_debug', 'font-subset' + EXE)
+if not os.path.isfile(FONT_SUBSET):
+  FONT_SUBSET = os.path.join(SRC_DIR, 'out', 'host_debug_unopt', 'font-subset' + EXE)
+if not os.path.isfile(FONT_SUBSET):
+  raise Exception('Could not locate font-subset%s in host_debug or host_debug_unopt - build before running this script.' % EXE)
+
+COMPARE_TESTS = (
+  (True,  '1.ttf', MATERIAL_TTF, [r'57347']),
+  (True,  '1.ttf', MATERIAL_TTF, [r'0xE003']),
+  (True,  '1.ttf', MATERIAL_TTF, [r'\uE003']),
+  (False, '1.ttf', MATERIAL_TTF, [r'57348']), # False because different codepoint
+  (True,  '2.ttf', MATERIAL_TTF, [r'0xE003', r'0xE004']),
+  (True,  '2.ttf', MATERIAL_TTF, [r'0xE003', r'0xE004', r'57347',]), # Duplicated codepoint
+  (True,  '3.ttf', MATERIAL_TTF, [r'0xE003', r'0xE004', r'0xE021',]),
+)
+
+FAIL_TESTS = [
+  ([FONT_SUBSET, 'output.ttf', 'does-not-exist.ttf'], ['1',]), # non-existant input font
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], ['0xFFFFFFFF',]), # Value too big.
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], ['-1',]), # invalid value
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], ['foo',]), # no valid values
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], ['0xE003', '0x12', '0xE004',]), # codepoint not in font
+  ([FONT_SUBSET, 'non-existant-dir/output.ttf', MATERIAL_TTF], ['0xE003',]), # dir doesn't exist
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], [' ',]), # empty input
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], []), # empty input
+  ([FONT_SUBSET, 'output.ttf', MATERIAL_TTF], ['']), # empty input
+]
+
+def RunCmd(cmd, codepoints, fail=False):
+  print('Running command:')
+  print('       %s' % ' '.join(cmd))
+  print('STDIN: "%s"' % ' '.join(codepoints))
+  p = subprocess.Popen(
+    cmd,
+    stdout=subprocess.PIPE,
+    stdin=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    cwd=SRC_DIR
+  )
+  stdout_data, stderr_data = p.communicate(input=' '.join(codepoints))
+  if p.returncode != 0 and fail == False:
+    print('FAILURE: %s' % p.returncode)
+    print('STDOUT:')
+    print(stdout_data)
+    print('STDERR:')
+    print(stderr_data)
+  elif p.returncode == 0 and fail == True:
+    print('FAILURE - test passed but should have failed.')
+    print('STDOUT:')
+    print(stdout_data)
+    print('STDERR:')
+    print(stderr_data)
+  else:
+    print('Success.')
+
+  return p.returncode
+
+
+def main():
+  print('Using font subset binary at %s' % FONT_SUBSET)
+  failures = 0
+  for should_pass, golden_font, input_font, codepoints in COMPARE_TESTS:
+    gen_ttf = os.path.join(SCRIPT_DIR, 'gen', golden_font)
+    golden_ttf = os.path.join(SCRIPT_DIR, 'fixtures', golden_font)
+    cmd = [FONT_SUBSET, gen_ttf, input_font]
+    RunCmd(cmd, codepoints)
+    cmp = filecmp.cmp(gen_ttf, golden_ttf, shallow=False)
+    if (should_pass and not cmp) or (not should_pass and cmp):
+      print('Test case %s failed.' % cmd)
+      failures += 1
+
+  with open(os.devnull, 'w') as devnull:
+    for cmd, codepoints in FAIL_TESTS:
+      if RunCmd(cmd, codepoints, fail=True) == 0:
+        failures += 1
+
+  if failures > 0:
+    print('%s test(s) failed.' % failures)
+    return 1
+
+  print('All tests passed')
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(main())
+