Add an option to use a prebuilt Dart SDK (flutter#26931)

.gitignore

@@ -130,3 +130,6 @@ app.*.symbols
 !**/ios/**/default.perspectivev3
 !/packages/flutter_tools/test/data/dart_dependencies_test/**/.packages
 !/dev/ci/**/Gemfile.lock
+
+# Prebuilt binaries.
+/prebuilts/

BUILD.gn

@@ -6,6 +6,7 @@ import("//flutter/common/config.gni")
 import("//flutter/shell/platform/config.gni")
 import("//flutter/shell/platform/glfw/config.gni")
 import("//flutter/testing/testing.gni")
+import("//third_party/dart/build/dart/copy_tree.gni")
 
 # Whether to build the dartdevc sdk, libraries, and source files
 # required for the flutter web sdk.
@@ -35,6 +36,35 @@ config("export_dynamic_symbols") {
   }
 }
 
+if (flutter_prebuilt_dart_sdk) {
+  copy_trees("_copy_trees") {
+    sources = [
+      {
+        target = "copy_dart_sdk"
+        visibility = [ ":dart_sdk" ]
+        source = target_prebuilt_dart_sdk
+        dest = "$root_out_dir/dart-sdk"
+        ignore_patterns = "{}"
+      },
+    ]
+  }
+}
+
+# Flutter SDK artifacts should only be built when either doing host builds, or
+# for cross-compiled desktop targets.
+_build_engine_artifacts =
+    current_toolchain == host_toolchain || (is_linux && !is_chromeos) || is_mac
+
+group("dart_sdk") {
+  if (_build_engine_artifacts) {
+    if (flutter_prebuilt_dart_sdk) {
+      public_deps = [ ":copy_dart_sdk" ]
+    } else {
+      public_deps = [ "//third_party/dart:create_sdk" ]
+    }
+  }
+}
+
 group("flutter") {
   testonly = true
 
@@ -44,22 +74,17 @@ group("flutter") {
     "//flutter/sky",
   ]
 
-  # Flutter SDK artifacts should only be built when either doing host builds, or
-  # for cross-compiled desktop targets.
-  build_engine_artifacts = current_toolchain == host_toolchain ||
-                           (is_linux && !is_chromeos) || is_mac
-
   # If enbaled, compile the SDK / snapshot.
   if (!is_fuchsia) {
     public_deps += [
       "//flutter/lib/snapshot:generate_snapshot_bin",
       "//flutter/lib/snapshot:kernel_platform_files",
     ]
 
-    if (build_engine_artifacts) {
+    if (_build_engine_artifacts) {
       public_deps += [
+        ":dart_sdk",
         "//flutter/flutter_frontend_server:frontend_server",
-        "//third_party/dart:create_sdk",
 
         # This must be listed explicitly for desktop cross-builds since
         # //flutter/lib/snapshot:generate_snapshot_bin will only build
@@ -73,7 +98,7 @@ group("flutter") {
     }
   }
 
-  if (build_engine_artifacts) {
+  if (_build_engine_artifacts) {
     public_deps += [
       "//flutter/shell/testing",
       "//flutter/tools/const_finder",

DEPS

@@ -713,6 +713,14 @@ hooks = [
       'src/build/win/generate_winrt_headers.py',
     ]
   },
+  {
+    'name': 'Download prebuilt Dart SDK',
+    'pattern': '.',
+    'action': [
+      'python3',
+      'src/flutter/tools/download_dart_sdk.py',
+    ]
+  },
   {
     'name': 'Setup githooks',
     'pattern': '.',

ci/licenses_golden/tool_signature

@@ -1,2 +1,2 @@
-Signature: b9105737a36020f19487fe9854f4be92
+Signature: 6f6132591430df787d08771477f3374d
 

common/config.gni

@@ -19,6 +19,9 @@ declare_args() {
 
   # Whether to use the Skia text shaper module for all text rendering
   flutter_always_use_skshaper = false
+
+  # Whether to use a prebuilt Dart SDK instead of building one.
+  flutter_prebuilt_dart_sdk = false
 }
 
 # feature_defines_list ---------------------------------------------------------
@@ -62,3 +65,17 @@ if (is_ios || is_mac) {
   flutter_cflags_objc_arc = flutter_cflags_objc + [ "-fobjc-arc" ]
   flutter_cflags_objcc_arc = flutter_cflags_objc_arc
 }
+
+# prebuilt Dart SDK location
+
+if (flutter_prebuilt_dart_sdk) {
+  _os_name = target_os
+  if (_os_name == "mac") {
+    _os_name = "macos"
+  } else if (_os_name == "win" || _os_name == "winuwp") {
+    _os_name = "windows"
+  }
+  target_prebuilt_dart_sdk =
+      "//flutter/prebuilts/$_os_name-$target_cpu/dart-sdk"
+  host_prebuilt_dart_sdk = "//flutter/prebuilts/$_os_name-$host_cpu/dart-sdk"
+}

testing/testing.gni

@@ -5,6 +5,7 @@
 import("//build/compiled_action.gni")
 import("//flutter/common/config.gni")
 import("//third_party/dart/build/dart/dart_action.gni")
+import("//third_party/dart/sdk_args.gni")
 
 is_aot_test =
     flutter_runtime_mode == "profile" || flutter_runtime_mode == "release"
@@ -56,18 +57,23 @@ template("dart_snapshot_kernel") {
   dart_action(target_name) {
     testonly = true
 
-    deps = [
-      # This generates the Frontend server snapshot used in this Dart action as
-      # well as the patched SDK.
-      "//third_party/dart/utils/kernel-service:frontend_server",
-    ]
+    if (flutter_prebuilt_dart_sdk) {
+      deps = [ "//flutter:dart_sdk" ]
+      script =
+          "$root_out_dir/dart-sdk/bin/snapshots/frontend_server.dart.snapshot"
+    } else {
+      deps = [
+        # This generates the Frontend server snapshot used in this Dart action as
+        # well as the patched SDK.
+        "//third_party/dart/utils/kernel-service:frontend_server",
+      ]
+      script = "$root_out_dir/frontend_server.dart.snapshot"
+    }
 
     if (is_aot_test) {
       deps += [ "//flutter/lib/snapshot:strong_platform" ]
     }
 
-    script = "$root_out_dir/frontend_server.dart.snapshot"
-
     inputs = [ invoker.dart_main ]
 
     outputs = [ invoker.dart_kernel ]

tools/download_dart_sdk.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# Copyright 2013 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# When the environment variable in FLUTTER_PREBUILTS_ENV_VAR below is defined
+# and is not '0' or 'false', this script downloads the Dart SDK that matches the
+# version in the source tree and puts it in prebuilts/.
+#
+# The return code of this script will always be 0, even if there is an error,
+# unless the --fail-loudly flag is passed.
+
+# TODO(zra): Eliminate this script and download through the DEPS file if/when
+# the Dart SDKs pulled by this script are uploaded to cipd.
+
+import argparse
+import os
+import multiprocessing
+import platform
+import re
+import shutil
+import subprocess
+import sys
+import zipfile
+
+
+SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+FLUTTER_DIR = os.path.join(SRC_ROOT, 'flutter')
+FLUTTER_PREBUILTS_DIR = os.path.join(FLUTTER_DIR, 'prebuilts')
+DART_DIR = os.path.join(SRC_ROOT, 'third_party', 'dart')
+DART_VERSION = os.path.join(DART_DIR, 'tools', 'VERSION')
+FLUTTER_PREBUILTS_ENV_VAR = 'FLUTTER_PREBUILT_DART_SDK'
+
+
+# The Dart SDK script is the source of truth about the sematic version.
+sys.path.append(os.path.join(DART_DIR, 'tools'))
+import utils
+
+
+# Prints to stderr.
+def eprint(*args, **kwargs):
+  print(*args, file=sys.stderr, **kwargs)
+
+
+# Try to guess the host operating system.
+def GuessOS():
+  os_name = utils.GuessOS()
+  if os_name == 'win32':
+    os_name = 'windows'
+  if os_name not in ['linux', 'macos', 'windows']:
+    eprint('Could not determine the OS: "%s"' % os_name)
+    return None
+  return os_name
+
+
+# For os `os_name` return a list of architectures for which prebuilts are
+# supported. Kepp in sync with `can_use_prebuilt_dart` in //flutter/tools/gn.
+def ArchitecturesForOS(os_name):
+  if os_name == 'linux':
+    return ['x64', 'arm64']
+  elif os_name == 'macos':
+    return ['x64', 'arm64']
+  elif os_name =='windows':
+    return ['x64']
+
+  eprint('Could not determine architectures for os "%s"' % os_name)
+  return None
+
+
+# Downloads a Dart SDK to //flutter/prebuilts.
+def DownloadDartSDK(channel, version, os_name, arch):
+  file = 'dartsdk-{}-{}-release.zip'.format(os_name, arch)
+  url = 'https://storage.googleapis.com/dart-archive/channels/{}/raw/{}/sdk/{}'.format(
+    channel, version, file,
+  )
+  dest = os.path.join(FLUTTER_PREBUILTS_DIR, file)
+
+  stamp_file = '{}.stamp'.format(dest)
+  version_stamp = None
+  try:
+    with open(stamp_file) as fd:
+      version_stamp = fd.read()
+  except:
+    version_stamp = 'none'
+
+  if version == version_stamp:
+    # The prebuilt Dart SDK is already up-to-date. Indicate that the download
+    # should be skipped by returning the empty string.
+    return ''
+
+  if os.path.isfile(dest):
+    os.unlink(dest)
+
+  curl_command = ['curl', url, '-o', dest]
+  curl_result = subprocess.run(
+    curl_command,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE,
+    universal_newlines=True,
+  )
+  if curl_result.returncode != 0:
+    eprint('Failed to download: stdout:\n{}\nstderr:\n{}'.format(
+      curl_result.stdout, curl_result.stderr,
+    ))
+    return None
+
+  return dest
+
+
+# A custom ZipFile class that preserves file permissions.
+class ZipFileWithPermissions(zipfile.ZipFile):
+  def _extract_member(self, member, targetpath, pwd):
+    if not isinstance(member, zipfile.ZipInfo):
+      member = self.getinfo(member)
+
+    targetpath = super()._extract_member(member, targetpath, pwd)
+
+    attr = member.external_attr >> 16
+    if attr != 0:
+      os.chmod(targetpath, attr)
+    return targetpath
+
+
+# Extracts a Dart SDK in //fluter/prebuilts
+def ExtractDartSDK(archive, os_name, arch):
+  os_arch = '{}-{}'.format(os_name, arch)
+  dart_sdk = os.path.join(FLUTTER_PREBUILTS_DIR, os_arch, 'dart-sdk')
+  if os.path.isdir(dart_sdk):
+    shutil.rmtree(dart_sdk)
+
+  extract_dest = os.path.join(FLUTTER_PREBUILTS_DIR, os_arch)
+  os.makedirs(extract_dest, exist_ok=True)
+
+  with ZipFileWithPermissions(archive, "r") as z:
+    z.extractall(extract_dest)
+
+
+def DownloadAndExtract(channel, version, os_name, arch):
+  archive = DownloadDartSDK(channel, version, os_name, arch)
+  if archive == None:
+    return 1
+  if archive == '':
+    return 0
+  ExtractDartSDK(archive, os_name, arch)
+  try:
+    stamp_file = '{}.stamp'.format(archive)
+    with open(stamp_file, "w") as fd:
+      fd.write(version)
+  except Exception as e:
+    eprint('Failed to write Dart SDK version stamp file:\n{}'.format(e))
+    return 1
+  return 0
+
+
+def Main():
+  parser = argparse.ArgumentParser()
+  parser.add_argument(
+    '--fail-loudly',
+    action='store_true',
+    default=False,
+    help="Return an error code if a prebuilt couldn't be fetched and extracted")
+  args = parser.parse_args()
+  fail_loudly = 1 if args.fail_loudly else 0
+
+  prebuilt_enabled = os.environ.get(FLUTTER_PREBUILTS_ENV_VAR, 'false')
+  if prebuilt_enabled == '0' or prebuilt_enabled.lower() == 'false':
+    return 0
+
+  os.makedirs(FLUTTER_PREBUILTS_DIR, exist_ok=True)
+
+  # Read //third_party/dart/tools/VERSION to extract information about the
+  # Dart SDK version.
+  version = utils.ReadVersionFile()
+  if version == None:
+    return fail_loudly
+  channel = version.channel
+
+  # A short Dart SDK version string used in the download url.
+  if channel == 'be':
+    dart_git_rev = utils.GetGitRevision()
+    semantic_version = 'hash/{}'.format(dart_git_rev)
+  semantic_version = utils.GetSemanticSDKVersion()
+
+  os_name = GuessOS()
+  if os_name == None:
+    return fail_loudly
+
+  architectures = ArchitecturesForOS(os_name)
+  if architectures == None:
+    return fail_loudly
+
+  # Download and extract variants in parallel
+  pool = multiprocessing.Pool()
+  tasks = [(channel, semantic_version, os_name, arch) for arch in architectures]
+  async_results = [pool.apply_async(DownloadAndExtract, t) for t in tasks]
+  success = True
+  for async_result in async_results:
+    result = async_result.get()
+    success = success and (result == 0)
+
+  return 0 if success else fail_loudly
+
+
+if __name__ == '__main__':
+  sys.exit(Main())