Use a pool for dart actions to avoid OOMs (flutter#27781)

* Use a pool for dart actions to avoid OOMs

* Add Windows support

build/concurrent_jobs.gni

@@ -0,0 +1,12 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+_script = "//flutter/build/get_concurrent_jobs.py"
+_args = [
+  "--reserve-memory=1GB",
+  "--memory-per-job",
+  "dart=1GB",
+]
+
+concurrent_jobs = exec_script(_script, _args, "json", [ _script ])

build/dart/BUILD.gn

@@ -0,0 +1,16 @@
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("//flutter/build/concurrent_jobs.gni")
+
+declare_args() {
+  # Maximum number of Dart processes to run in parallel.
+  #
+  # To avoid out-of-memory errors we explicitly reduce the number of jobs.
+  concurrent_dart_jobs = concurrent_jobs.dart
+}
+
+pool("dart_pool") {
+  depth = concurrent_dart_jobs
+}

build/get_concurrent_jobs.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+# Copyright 2019 The Fuchsia Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This script computes the number of concurrent jobs that can run in the
+# build as a function of the machine. It accepts a set of key value pairs
+# given by repeated --memory-per-job arguments. For example:
+#
+# $ get_concurrent_jobs.py --memory-per-job dart=1GB
+#
+# The result is a json map printed to stdout that gives the number of
+# concurrent jobs allowed of each kind. For example:
+#
+# {"dart": 8}
+#
+# Some memory can be held out of the calculation with the --reserve-memory flag.
+
+import argparse
+import ctypes
+import json
+import multiprocessing
+import os
+import re
+import subprocess
+import sys
+
+UNITS = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40}
+
+
+# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
+# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
+class MEMORYSTATUSEX(ctypes.Structure):
+  _fields_ = [
+      ("dwLength", ctypes.c_ulong),
+      ("dwMemoryLoad", ctypes.c_ulong),
+      ("ullTotalPhys", ctypes.c_ulonglong),
+      ("ullAvailPhys", ctypes.c_ulonglong),
+      ("ullTotalPageFile", ctypes.c_ulonglong),
+      ("ullAvailPageFile", ctypes.c_ulonglong),
+      ("ullTotalVirtual", ctypes.c_ulonglong),
+      ("ullAvailVirtual", ctypes.c_ulonglong),
+      ("sullAvailExtendedVirtual", ctypes.c_ulonglong),
+  ]
+
+
+def GetTotalMemory():
+  if sys.platform in ('win32', 'cygwin'):
+    stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
+    success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
+    return stat.ullTotalPhys if success else 0
+  elif sys.platform.startswith('linux'):
+    if os.path.exists("/proc/meminfo"):
+      with open("/proc/meminfo") as meminfo:
+        memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
+        for line in meminfo:
+          match = memtotal_re.match(line)
+          if match:
+            return float(match.group(1)) * 2**10
+  elif sys.platform == 'darwin':
+    try:
+      return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
+    except Exception:
+      return 0
+  else:
+    return 0
+
+
+def ParseSize(string):
+  i = next(i for (i, c) in enumerate(string) if not c.isdigit())
+  number = string[:i].strip()
+  unit = string[i:].strip()
+  return int(float(number) * UNITS[unit])
+
+
+class ParseSizeAction(argparse.Action):
+
+  def __call__(self, parser, args, values, option_string=None):
+    sizes = getattr(args, self.dest, [])
+    for value in values:
+      (k, v) = value.split('=', 1)
+      sizes.append((k, ParseSize(v)))
+    setattr(args, self.dest, sizes)
+
+
+def Main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+      '--memory-per-job',
+      action=ParseSizeAction,
+      default=[],
+      nargs='*',
+      help='Key value pairings (dart=1GB) giving an estimate of the amount of '
+           'memory needed for the class of job.')
+    parser.add_argument(
+      '--reserve-memory',
+      type=ParseSize,
+      default=0,
+      help='The amount of memory to be held out of the amount for jobs to use.')
+    args = parser.parse_args()
+
+    total_memory = GetTotalMemory()
+
+    # Ensure the total memory used in the calculation below is at least 0
+    mem_total_bytes = max(0, total_memory - args.reserve_memory)
+
+    # Ensure the number of cpus used in the calculation below is at least 1
+    try:
+      cpu_cap = multiprocessing.cpu_count()
+    except:
+      cpu_cap = 1
+
+    concurrent_jobs = {}
+    for job, memory_per_job in args.memory_per_job:
+      # Calculate the number of jobs that will fit in memory. Ensure the
+      # value is at least 1.
+      num_concurrent_jobs = int(max(1, mem_total_bytes / memory_per_job))
+      # Cap the number of jobs by the number of cpus available.
+      concurrent_jobs[job] = min(num_concurrent_jobs, cpu_cap)
+
+    print(json.dumps(concurrent_jobs))
+
+    return 0
+
+
+if __name__ == '__main__':
+  sys.exit(Main())

flutter_frontend_server/BUILD.gn

@@ -23,6 +23,8 @@ frontend_server_files +=
 application_snapshot("frontend_server") {
   main_dart = "bin/starter.dart"
   deps = [ "//flutter/lib/snapshot:kernel_platform_files" ]
+  pool = "//flutter/build/dart:dart_pool"
+
   dot_packages = rebase_path(".dart_tool/package_config.json")
   flutter_patched_sdk = rebase_path("$root_out_dir/flutter_patched_sdk")
   training_args = [

lib/snapshot/BUILD.gn

@@ -243,6 +243,8 @@ compile_platform("strong_platform") {
     "$root_out_dir/flutter_patched_sdk/vm_outline_strong.dill",
   ]
 
+  pool = "//flutter/build/dart:dart_pool"
+
   is_runtime_mode_release =
       flutter_runtime_mode == "release" || flutter_runtime_mode == "jit_release"
   args = [

testing/dart/compile_test.gni

@@ -30,6 +30,7 @@ template("compile_flutter_dart_test") {
       "//flutter/flutter_frontend_server:frontend_server",
       "//flutter/lib/snapshot:strong_platform",
     ]
+    pool = "//flutter/build/dart:dart_pool"
     script = "$root_gen_dir/frontend_server.dart.snapshot"
     packages = rebase_path(invoker.packages)  # rebase_path(".packages")
 

testing/testing.gni

@@ -57,6 +57,7 @@ template("_frontend_server") {
       inputs = invoker.inputs
       outputs = invoker.outputs
       depfile = invoker.depfile
+      pool = "//flutter/build/dart:dart_pool"
 
       ext = ""
       if (is_win) {
@@ -76,6 +77,7 @@ template("_frontend_server") {
       forward_variables_from(invoker, "*")
       deps += [ "//third_party/dart/utils/kernel-service:frontend_server" ]
       script = "$root_out_dir/frontend_server.dart.snapshot"
+      pool = "//flutter/build/dart:dart_pool"
     }
   }
 }
@@ -157,6 +159,8 @@ template("dart_snapshot_aot") {
 
     tool = "//third_party/dart/runtime/bin:gen_snapshot"
 
+    pool = "//flutter/build/dart:dart_pool"
+
     inputs = [ invoker.dart_kernel ]
 
     # Custom ELF loader is used for Mac and Windows.

tools/const_finder/BUILD.gn

@@ -8,6 +8,7 @@ application_snapshot("const_finder") {
   main_dart = "bin/main.dart"
   dot_packages = ".dart_tool/package_config.json"
   dart_snapshot_kind = "kernel"
+  pool = "//flutter/build/dart:dart_pool"
 
   training_args = [ "--help" ]
 

web_sdk/BUILD.gn

@@ -4,7 +4,6 @@
 
 import("//flutter/common/config.gni")
 import("//third_party/dart/build/dart/dart_action.gni")
-import("//third_party/dart/utils/compile_platform.gni")
 
 sdk_dill = "$root_out_dir/flutter_web_sdk/kernel/flutter_ddc_sdk.dill"
 sdk_dill_sound =
@@ -51,6 +50,7 @@ prebuilt_dart_action("web_ui_sources") {
   script = "sdk_rewriter.dart"
   output_dir = rebase_path("$root_out_dir/flutter_web_sdk/lib/ui/")
   input_dir = rebase_path("//flutter/lib/web_ui/lib/")
+  pool = "//flutter/build/dart:dart_pool"
 
   outputs = [ "$target_gen_dir/$target_name.stamp" ]
 
@@ -75,6 +75,7 @@ prebuilt_dart_action("web_engine_sources") {
   script = "sdk_rewriter.dart"
   output_dir = rebase_path("$root_out_dir/flutter_web_sdk/lib/_engine/")
   input_dir = rebase_path("//flutter/lib/web_ui/lib/src/")
+  pool = "//flutter/build/dart:dart_pool"
 
   outputs = [ "$target_gen_dir/$target_name.stamp" ]
 
@@ -122,6 +123,8 @@ template("_dartdevc") {
       inputs = invoker.inputs
       outputs = invoker.outputs
 
+      pool = "//flutter/build/dart:dart_pool"
+
       ext = ""
       if (is_win) {
         ext = ".exe"
@@ -149,6 +152,8 @@ template("_dartdevc") {
       ]
 
       script = "//third_party/dart/pkg/dev_compiler/bin/dartdevc.dart"
+
+      pool = "//flutter/build/dart:dart_pool"
     }
   }
 }
@@ -176,6 +181,8 @@ template("_kernel_worker") {
       inputs = invoker.inputs
       outputs = invoker.outputs
 
+      pool = "//flutter/build/dart:dart_pool"
+
       ext = ""
       if (is_win) {
         ext = ".exe"
@@ -204,6 +211,8 @@ template("_kernel_worker") {
       ]
 
       script = "//third_party/dart/utils/bazel/kernel_worker.dart"
+
+      pool = "//flutter/build/dart:dart_pool"
     }
   }
 }