Reverts "Refactors RBE support" (flutter#49654)

Reverts flutter#49416 Initiated by: zanderso This change reverts the following previous change: Original Description: This PR refactors the RBE build and adds support for mac-hosted builds to use RBE. In particular, configurations files are downloaded from flutter-internal CIPD instead of being included in the repo. The download from CIPD is triggered by the presence of the `use_rbe` flag as a custom var in the `.gclient` file. This PR also turns on RBE for Linux and macOS builds that do not produce artifacts. flutter/buildroot#807
faco · Jan 9, 2024 · 80a984a · 80a984a
1 parent 555aa9b
commit 80a984a
Show file tree

Hide file tree

Showing 7 changed files with 25 additions and 135 deletions.
diff --git a/.gitignore b/.gitignore
@@ -130,9 +130,4 @@ app.*.symbols
 
 # Prebuilt binaries.
 /prebuilts/
-
-# GN build support for protobufs vended by Fuchsia.
 /build/secondary/third_party/protobuf
-
-# RBE support configurations and scripts vended from CIPD
-/build/rbe
diff --git a/DEPS b/DEPS
@@ -54,7 +54,7 @@ vars = {
   # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
   'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
 
-  'reclient_version': 'git_revision:f3883c2237b0eb9cc9524cb571b5ab8378f257e4',
+  'reclient_version': 'git_revision:81e819b39d4743462857cc55430d898b9fcca1af',
 
   'gcloud_version': 'version:[email protected]',
 
@@ -127,10 +127,6 @@ vars = {
   # specified by GOMA_DIR, or installed in the default goma install location.
   'use_cipd_goma': False,
 
-  # When this is true, the Flutter Engine's configuration files and scripts for
-  # RBE will be downloaded from CIPD. This option is only usable by Googlers.
-  'use_rbe': False,
-
   # This is not downloaded be default because it increases the
   # `gclient sync` time by between 1 and 3 minutes. This option is enabled
   # in flutter/ci/builders/mac_impeller_cmake_example.json, and is likely to
@@ -271,7 +267,7 @@ allowed_hosts = [
 ]
 
 deps = {
-  'src': 'https://github.com/flutter/buildroot.git' + '@' + 'a43582b52d361bc3da156a8e6eab6dd947ca339d',
+  'src': 'https://github.com/flutter/buildroot.git' + '@' + '8c274b21f1ad4f2aec0a5e0ae8f4264393045b4b',
 
   'src/flutter/third_party/rapidjson':
    Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b',
@@ -848,7 +844,7 @@ deps = {
         'version': Var('clang_version'),
       }
     ],
-    'condition': 'host_os == "linux" or host_os == "mac"',
+    'condition': 'host_os == "linux" and host_cpu == "x64"',
     'dep_type': 'cipd',
   },
 
@@ -908,48 +904,15 @@ deps = {
     'dep_type': 'cipd',
   },
 
-  # RBE binaries and configs.
+  # reclient.
   'src/buildtools/linux-x64/reclient': {
     'packages': [
       {
         'package': 'infra/rbe/client/${{platform}}',
         'version': Var('reclient_version'),
       }
     ],
-    'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
-    'dep_type': 'cipd',
-  },
-
-  'src/buildtools/mac-arm64/reclient': {
-    'packages': [
-      {
-        'package': 'infra/rbe/client/${{platform}}',
-        'version': Var('reclient_version'),
-      }
-    ],
-    'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
-    'dep_type': 'cipd',
-  },
-
-  'src/buildtools/mac-x64/reclient': {
-    'packages': [
-      {
-        'package': 'infra/rbe/client/${{platform}}',
-        'version': Var('reclient_version'),
-      }
-    ],
-    'condition': 'use_rbe and host_os == "mac" and host_cpu == "x64"',
-    'dep_type': 'cipd',
-  },
-
-  'src/flutter/build/rbe': {
-    'packages': [
-      {
-        'package': 'flutter_internal/rbe/reclient_cfgs',
-        'version': 'U42C0v8jI-_YREjd8rbDEt0evvqvLWJ_NTkaiJ_Clt8C',
-      }
-    ],
-    'condition': 'use_rbe',
+    'condition': 'host_os == "linux" and host_cpu == "x64"',
     'dep_type': 'cipd',
   },
 
@@ -961,18 +924,7 @@ deps = {
         'version': Var('gcloud_version'),
       }
     ],
-    'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
-    'dep_type': 'cipd',
-  },
-
-  'src/buildtools/mac-arm64/gcloud': {
-    'packages': [
-      {
-        'package': 'infra/3pp/tools/gcloud/${{platform}}',
-        'version': Var('gcloud_version'),
-      }
-    ],
-    'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
+    'condition': 'host_os == "linux" and host_cpu == "x64"',
     'dep_type': 'cipd',
   },
 

diff --git a/ci/builders/linux_android_emulator.json b/ci/builders/linux_android_emulator.json
@@ -7,9 +7,6 @@
                 "kvm=1",
                 "cores=8"
             ],
-            "gclient_variables": {
-                "use_rbe": true
-            },
             "gn": [
                 "--android",
                 "--android-cpu=x64",
@@ -72,9 +69,6 @@
                 "kvm=1",
                 "cores=8"
             ],
-            "gclient_variables": {
-                "use_rbe": true
-            },
             "gn": [
                 "--android",
                 "--android-cpu=x86",

diff --git a/ci/builders/linux_unopt.json b/ci/builders/linux_unopt.json
@@ -6,9 +6,6 @@
                 "os=Linux",
                 "cores=32"
             ],
-            "gclient_variables": {
-                "use_rbe": true
-            },
             "gn": [
                 "--runtime-mode",
                 "debug",
@@ -87,9 +84,6 @@
                 "os=Linux"
             ],
             "dependencies": [],
-            "gclient_variables": {
-                "use_rbe": true
-            },
             "gn": [
                 "--android",
                 "--embedder-for-target",
@@ -118,9 +112,6 @@
                     "version": "last_updated:2023-02-03T15:32:01-0800"
                 }
             ],
-            "gclient_variables": {
-                "use_rbe": true
-            },
             "gn": [
                 "--android",
                 "--unoptimized",

diff --git a/ci/builders/mac_unopt.json b/ci/builders/mac_unopt.json
@@ -16,19 +16,15 @@
                 "mac_model=Macmini8,1"
             ],
             "gclient_variables": {
-                "download_android_deps": false,
-                "use_rbe": true
+                "download_android_deps": false
             },
             "gn": [
                 "--runtime-mode",
                 "debug",
                 "--unoptimized",
                 "--no-lto",
                 "--prebuilt-dart-sdk",
-                "--enable-impeller-3d",
-                "--rbe",
-                "--no-goma",
-                "--xcode-symlinks"
+                "--enable-impeller-3d"
             ],
             "name": "host_debug_unopt",
             "ninja": {
@@ -79,18 +75,14 @@
                 "cpu=x86"
             ],
             "gclient_variables": {
-                "download_android_deps": false,
-                "use_rbe": true
+                "download_android_deps": false
             },
             "gn": [
                 "--ios",
                 "--runtime-mode",
                 "debug",
                 "--simulator",
-                "--no-lto",
-                "--rbe",
-                "--no-goma",
-                "--xcode-symlinks"
+                "--no-lto"
             ],
             "name": "ios_debug_sim",
             "ninja": {
@@ -140,8 +132,7 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false,
-                "use_rbe": true
+                "download_android_deps": false
             },
             "gn": [
                "--runtime-mode",
@@ -151,10 +142,7 @@
                "--prebuilt-dart-sdk",
                "--force-mac-arm64",
                "--mac-cpu",
-               "arm64",
-               "--rbe",
-               "--no-goma",
-               "--xcode-symlinks"
+               "arm64"
             ],
             "name": "host_debug_unopt_arm64",
             "ninja": {
@@ -193,8 +181,7 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false,
-                "use_rbe": true
+                "download_android_deps": false
             },
             "gn": [
                 "--ios",
@@ -204,10 +191,7 @@
                 "--no-lto",
                 "--force-mac-arm64",
                 "--simulator-cpu",
-                "arm64",
-                "--rbe",
-                "--no-goma",
-                "--xcode-symlinks"
+                "arm64"
             ],
             "name": "ios_debug_sim_arm64",
             "ninja": {
@@ -267,8 +251,7 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false,
-                "use_rbe": true
+                "download_android_deps": false
             },
             "gn": [
                 "--ios",
@@ -279,10 +262,7 @@
                 "--force-mac-arm64",
                 "--simulator-cpu",
                 "arm64",
-                "--darwin-extension-safe",
-                "--rbe",
-                "--no-goma",
-                "--xcode-symlinks"
+                "--darwin-extension-safe"
             ],
             "name": "ios_debug_sim_arm64_extension_safe",
             "ninja": {

diff --git a/ci/builders/standalone/linux_benchmarks.json b/ci/builders/standalone/linux_benchmarks.json
@@ -4,8 +4,7 @@
         "os=Linux"
     ],
     "gclient_variables": {
-        "download_android_deps": false,
-        "use_rbe": true
+        "download_android_deps": false
     },
     "gn": [
         "--runtime-mode",

diff --git a/tools/gn b/tools/gn
@@ -205,6 +205,8 @@ def buildtools_dir():
   host_cpu = get_host_cpu()
   if host_os == 'win':
     host_os = 'windows'
+  if host_os == 'mac' and host_cpu == 'arm64':
+    host_cpu = 'x64'
   return '%s-%s' % (host_os, host_cpu)
 
 
@@ -215,10 +217,9 @@ def setup_rbe(args):
   if not args.rbe:
     return rbe_gn_args
 
-  if get_host_os() not in ['linux', 'mac']:
+  if get_host_os() not in ['linux']:
     print(
-        'The --rbe flag has no effect. RBE is currently only supported on '
-        'macOS and Linux.'
+        'The --rbe flag has no effect. RBE is currently only supported on Linux.'
     )
     return rbe_gn_args
 
@@ -238,9 +239,7 @@ def setup_rbe(args):
     )
     bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap')
     reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy')
-    rbe_cfg_path = os.path.join(
-        SRC_ROOT, 'flutter', 'build', 'rbe', 'reclient.cfg'
-    )
+    rbe_cfg_path = os.path.join(SRC_ROOT, 'build', 'rbe.cfg')
     bootstrap_cmd = [
         bootstrap_path,
         '--re_proxy=' + reproxy_path,
@@ -261,20 +260,8 @@ def setup_rbe(args):
     rbe_gn_args['rbe_dial_timeout'] = args.rbe_dial_timeout
   if args.rbe_platform:
     rbe_gn_args['rbe_platform'] = args.rbe_platform
-
-  rbe_gn_args['rbe_dir'] = os.path.join(
-      SRC_ROOT, 'buildtools', buildtools_dir(), 'reclient'
-  )
-
-  rbe_gn_args['rbe_cfg'] = os.path.join(
-      SRC_ROOT, 'flutter', 'build', 'rbe',
-      'rewrapper-' + buildtools_dir() + '.cfg'
-  )
-
-  if sys.platform == 'darwin':
-    if (not running_on_luci or args.xcode_symlinks or
-        os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
-      rbe_gn_args['create_xcode_symlinks'] = True
+  if args.rbe_dir:
+    rbe_gn_args['rbe_dir'] = args.rbe_dir
 
   return rbe_gn_args
 
@@ -298,16 +285,8 @@ def setup_goma(args):
   # care of starting and stopping the compiler proxy.
   running_on_luci = os.environ.get('LUCI_CONTEXT') is not None
 
-  # The GOMA client has no arm64 binary, so run the x64 binary through
-  # Rosetta.
-  buildtools_platform = buildtools_dir()
-  if buildtools_platform == 'mac-arm64':
-    buildtools_platform = 'mac-x64'
-
   # Prefer the goma fetched by gclient if it exists.
-  cipd_goma_dir = os.path.join(
-      SRC_ROOT, 'buildtools', buildtools_platform, 'goma'
-  )
+  cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
 
   # Next, if GOMA_DIR is set, use that install.
   goma_dir = os.environ.get('GOMA_DIR')