Implement asynchronous texture uploads when using the Metal backend o…

…n iOS. (flutter#17046) This moves the Metal `GrContext` creation utilities from `GPUSurfaceMetal` into a separate `IOSContext` object subclass. An analogue of this object was used in the GL regime for the management of onscreen and offscreen contexts that were not tied to the lifecycle of the `GPUSurface`. This pattern has now been generalized for use with all backends that need a resource context (`IOSContextGL` and `IOContextMetal`). The platform views controller management in the `ExternalViewEmbedder` interface implementation was repeated three times for [Metal][metal], [OpenGL](opengl) and [Software](software) rendering. This repetition has been removed and a single implementation present in the base `IOSSurface` and used on all platforms. Addition of new client rendering APIs should not affect how the engine renders into the platform view interleaving levels. All rendering API selection logic has been moved into a single set of utilities in `rendering_api_selection.h`. This enables the removal of a lot of code blocks guarded by `FLUTTER_SHELL_ENABLE_METAL`. The remaining uses of this will be removed when unified builds are enabled. The Metal backend now also adds traces similar to the GL backend. The `IOGLContext` has been renamed to `IOContextGL` to be more in line with the convention used in this library. Fixes flutter/flutter#41827 Adds flutter/flutter#52150 [metal]: https://github.com/flutter/engine/blob/1194ba2b218706a201c5d2c5325b55a5932546c5/shell/platform/darwin/ios/ios_surface_metal.mm#L55 [opengl]: https://github.com/flutter/engine/blob/1194ba2b218706a201c5d2c5325b55a5932546c5/shell/platform/darwin/ios/ios_surface_gl.mm#L95 [software]: https://github.com/flutter/engine/blob/1194ba2b218706a201c5d2c5325b55a5932546c5/shell/platform/darwin/ios/ios_surface_software.mm#L146
jwise · Mar 10, 2020 · 17e07c5 · 17e07c5
1 parent f1ff6a2
commit 17e07c5
Show file tree

Hide file tree

Showing 40 changed files with 1,031 additions and 779 deletions.
diff --git a/ci/licenses_golden/licenses_flutter b/ci/licenses_golden/licenses_flutter
@@ -868,12 +868,18 @@ FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/platform_messa
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.h
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/Source/vsync_waiter_ios.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/framework/module.modulemap
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_gl.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_gl.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_metal.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_metal.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_context_software.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_external_texture_gl.mm
-FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_context.h
-FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_context.mm
-FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_render_target.h
-FILE: ../../../flutter/shell/platform/darwin/ios/ios_gl_render_target.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.h
+FILE: ../../../flutter/shell/platform/darwin/ios/ios_render_target_gl.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_gl.h
@@ -884,6 +890,8 @@ FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_software.h
 FILE: ../../../flutter/shell/platform/darwin/ios/ios_surface_software.mm
 FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.h
 FILE: ../../../flutter/shell/platform/darwin/ios/platform_view_ios.mm
+FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.h
+FILE: ../../../flutter/shell/platform/darwin/ios/rendering_api_selection.mm
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/FlutterMacOS.podspec
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterAppDelegate.h
 FILE: ../../../flutter/shell/platform/darwin/macos/framework/Headers/FlutterDartProject.h

diff --git a/flow/compositor_context.cc b/flow/compositor_context.cc
@@ -70,6 +70,7 @@ CompositorContext::ScopedFrame::~ScopedFrame() {
 RasterStatus CompositorContext::ScopedFrame::Raster(
     flutter::LayerTree& layer_tree,
     bool ignore_raster_cache) {
+  TRACE_EVENT0("flutter", "CompositorContext::ScopedFrame::Raster");
   bool root_needs_readback = layer_tree.Preroll(*this, ignore_raster_cache);
   bool needs_save_layer = root_needs_readback && !surface_supports_readback();
   PostPrerollResult post_preroll_result = PostPrerollResult::kSuccess;

diff --git a/lib/ui/painting/image_decoder.cc b/lib/ui/painting/image_decoder.cc
@@ -282,6 +282,7 @@ static SkiaGPUObject<SkImage> UploadRasterImage(
           })
           .SetIfFalse([&result, context = io_manager->GetResourceContext(),
                        &pixmap, queue = io_manager->GetSkiaUnrefQueue()] {
+            TRACE_EVENT0("flutter", "MakeCrossContextImageFromPixmap");
             sk_sp<SkImage> texture_image = SkImage::MakeCrossContextFromPixmap(
                 context.get(),  // context
                 pixmap,         // pixmap

diff --git a/shell/common/rasterizer.cc b/shell/common/rasterizer.cc
@@ -295,6 +295,7 @@ RasterStatus Rasterizer::DoDraw(
 }
 
 RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
+  TRACE_EVENT0("flutter", "Rasterizer::DrawToSurface");
   FML_DCHECK(surface_);
 
   auto frame = surface_->AcquireFrame(layer_tree.frame_size());
@@ -350,6 +351,7 @@ RasterStatus Rasterizer::DrawToSurface(flutter::LayerTree& layer_tree) {
     FireNextFrameCallbackIfPresent();
 
     if (surface_->GetContext()) {
+      TRACE_EVENT0("flutter", "PerformDeferredSkiaCleanup");
       surface_->GetContext()->performDeferredCleanup(kSkiaCleanupExpiration);
     }
 

diff --git a/shell/gpu/gpu_surface_gl.h b/shell/gpu/gpu_surface_gl.h
@@ -26,6 +26,7 @@ class GPUSurfaceGL : public Surface {
                GPUSurfaceGLDelegate* delegate,
                bool render_to_surface);
 
+  // |Surface|
   ~GPUSurfaceGL() override;
 
   // |Surface|

diff --git a/shell/gpu/gpu_surface_metal.h b/shell/gpu/gpu_surface_metal.h
@@ -19,11 +19,12 @@ namespace flutter {
 
 class GPUSurfaceMetal : public Surface {
  public:
-  GPUSurfaceMetal(GPUSurfaceDelegate* delegate, fml::scoped_nsobject<CAMetalLayer> layer);
   GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
-                  sk_sp<GrContext> gr_context,
-                  fml::scoped_nsobject<CAMetalLayer> layer);
+                  fml::scoped_nsobject<CAMetalLayer> layer,
+                  sk_sp<GrContext> context,
+                  fml::scoped_nsprotocol<id<MTLCommandQueue>> command_queue);
 
+  // |Surface|
   ~GPUSurfaceMetal() override;
 
  private:

diff --git a/shell/gpu/gpu_surface_metal.mm b/shell/gpu/gpu_surface_metal.mm
@@ -6,6 +6,7 @@
 
 #include <QuartzCore/CAMetalLayer.h>
 
+#include "flutter/fml/trace_event.h"
 #include "third_party/skia/include/core/SkSurface.h"
 #include "third_party/skia/include/gpu/GrBackendSurface.h"
 #include "third_party/skia/include/ports/SkCFObject.h"
@@ -15,62 +16,17 @@
 namespace flutter {
 
 GPUSurfaceMetal::GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
-                                 fml::scoped_nsobject<CAMetalLayer> layer)
-    : delegate_(delegate), layer_(std::move(layer)) {
-  if (!layer_) {
-    FML_LOG(ERROR) << "Could not create metal surface because of invalid layer.";
-    return;
-  }
-
-  layer.get().pixelFormat = MTLPixelFormatBGRA8Unorm;
+                                 fml::scoped_nsobject<CAMetalLayer> layer,
+                                 sk_sp<GrContext> context,
+                                 fml::scoped_nsprotocol<id<MTLCommandQueue>> command_queue)
+    : delegate_(delegate),
+      layer_(std::move(layer)),
+      context_(std::move(context)),
+      command_queue_(std::move(command_queue)) {
+  layer_.get().pixelFormat = MTLPixelFormatBGRA8Unorm;
   // Flutter needs to read from the color attachment in cases where there are effects such as
   // backdrop filters.
-  layer.get().framebufferOnly = NO;
-
-  auto metal_device = fml::scoped_nsprotocol<id<MTLDevice>>([layer_.get().device retain]);
-  auto metal_queue = fml::scoped_nsprotocol<id<MTLCommandQueue>>([metal_device newCommandQueue]);
-
-  if (!metal_device || !metal_queue) {
-    FML_LOG(ERROR) << "Could not create metal device or queue.";
-    return;
-  }
-
-  command_queue_ = metal_queue;
-
-  // The context creation routine accepts arguments using transfer semantics.
-  auto context = GrContext::MakeMetal(metal_device.release(), metal_queue.release());
-  if (!context) {
-    FML_LOG(ERROR) << "Could not create Skia metal context.";
-    return;
-  }
-
-  context_ = context;
-}
-
-GPUSurfaceMetal::GPUSurfaceMetal(GPUSurfaceDelegate* delegate,
-                                 sk_sp<GrContext> gr_context,
-                                 fml::scoped_nsobject<CAMetalLayer> layer)
-    : delegate_(delegate), layer_(std::move(layer)), context_(gr_context) {
-  if (!layer_) {
-    FML_LOG(ERROR) << "Could not create metal surface because of invalid layer.";
-    return;
-  }
-  if (!context_) {
-    FML_LOG(ERROR) << "Could not create metal surface because of invalid Skia metal context.";
-    return;
-  }
-
-  layer.get().pixelFormat = MTLPixelFormatBGRA8Unorm;
-
-  auto metal_device = fml::scoped_nsprotocol<id<MTLDevice>>([layer_.get().device retain]);
-  auto metal_queue = fml::scoped_nsprotocol<id<MTLCommandQueue>>([metal_device newCommandQueue]);
-
-  if (!metal_device || !metal_queue) {
-    FML_LOG(ERROR) << "Could not create metal device or queue.";
-    return;
-  }
-
-  command_queue_ = metal_queue;
+  layer_.get().framebufferOnly = NO;
 }
 
 GPUSurfaceMetal::~GPUSurfaceMetal() {
@@ -95,13 +51,20 @@
   }
 
   const auto bounds = layer_.get().bounds.size;
-  if (bounds.width <= 0.0 || bounds.height <= 0.0) {
+  const auto scale = layer_.get().contentsScale;
+  if (bounds.width <= 0.0 || bounds.height <= 0.0 || scale <= 0.0) {
     FML_LOG(ERROR) << "Metal layer bounds were invalid.";
     return nullptr;
   }
 
+  const auto scaled_bounds = CGSizeMake(bounds.width * scale, bounds.height * scale);
+
   ReleaseUnusedDrawableIfNecessary();
 
+  if (!CGSizeEqualToSize(scaled_bounds, layer_.get().drawableSize)) {
+    layer_.get().drawableSize = scaled_bounds;
+  }
+
   auto surface = SkSurface::MakeFromCAMetalLayer(context_.get(),            // context
                                                  layer_.get(),              // layer
                                                  kTopLeft_GrSurfaceOrigin,  // origin
@@ -118,6 +81,7 @@
   }
 
   auto submit_callback = [this](const SurfaceFrame& surface_frame, SkCanvas* canvas) -> bool {
+    TRACE_EVENT0("flutter", "GPUSurfaceMetal::Submit");
     canvas->flush();
 
     if (next_drawable_ == nullptr) {
@@ -159,9 +123,7 @@
 SkMatrix GPUSurfaceMetal::GetRootTransformation() const {
   // This backend does not currently support root surface transformations. Just
   // return identity.
-  SkMatrix matrix;
-  matrix.reset();
-  return matrix;
+  return {};
 }
 
 // |Surface|

diff --git a/shell/platform/darwin/ios/BUILD.gn b/shell/platform/darwin/ios/BUILD.gn
@@ -44,6 +44,8 @@ shared_library("create_flutter_framework_dylib") {
 
   ldflags = [ "-Wl,-install_name,@rpath/Flutter.framework/Flutter" ]
 
+  public = _flutter_framework_headers
+
   sources = [
     "framework/Source/FlutterAppDelegate.mm",
     "framework/Source/FlutterBinaryMessengerRelay.mm",
@@ -81,12 +83,16 @@ shared_library("create_flutter_framework_dylib") {
     "framework/Source/platform_message_router.mm",
     "framework/Source/vsync_waiter_ios.h",
     "framework/Source/vsync_waiter_ios.mm",
+    "ios_context.h",
+    "ios_context.mm",
+    "ios_context_gl.h",
+    "ios_context_gl.mm",
+    "ios_context_software.h",
+    "ios_context_software.mm",
     "ios_external_texture_gl.h",
     "ios_external_texture_gl.mm",
-    "ios_gl_context.h",
-    "ios_gl_context.mm",
-    "ios_gl_render_target.h",
-    "ios_gl_render_target.mm",
+    "ios_render_target_gl.h",
+    "ios_render_target_gl.mm",
     "ios_surface.h",
     "ios_surface.mm",
     "ios_surface_gl.h",
@@ -95,15 +101,10 @@ shared_library("create_flutter_framework_dylib") {
     "ios_surface_software.mm",
     "platform_view_ios.h",
     "platform_view_ios.mm",
+    "rendering_api_selection.h",
+    "rendering_api_selection.mm",
   ]
 
-  if (shell_enable_metal) {
-    sources += [
-      "ios_surface_metal.h",
-      "ios_surface_metal.mm",
-    ]
-  }
-
   sources += _flutter_framework_headers
 
   deps = [
@@ -126,6 +127,13 @@ shared_library("create_flutter_framework_dylib") {
 
   if (shell_enable_metal) {
     defines += [ "FLUTTER_SHELL_ENABLE_METAL=1" ]
+
+    sources += [
+      "ios_context_metal.h",
+      "ios_context_metal.mm",
+      "ios_surface_metal.h",
+      "ios_surface_metal.mm",
+    ]
   }
 
   libs = [

diff --git a/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm b/shell/platform/darwin/ios/framework/Source/FlutterEngine.mm
@@ -26,6 +26,7 @@
 #import "flutter/shell/platform/darwin/ios/framework/Source/platform_message_response_darwin.h"
 #import "flutter/shell/platform/darwin/ios/ios_surface.h"
 #import "flutter/shell/platform/darwin/ios/platform_view_ios.h"
+#include "flutter/shell/platform/darwin/ios/rendering_api_selection.h"
 
 NSString* const FlutterDefaultDartEntrypoint = nil;
 
@@ -427,7 +428,7 @@ - (BOOL)createShell:(NSString*)entrypoint libraryURI:(NSString*)libraryURI {
   }
 
   const auto threadLabel = [NSString stringWithFormat:@"%@.%zu", _labelPrefix, shellCount++];
-  FML_DLOG(INFO) << "Creating threadHost for " << threadLabel.UTF8String;
+
   // The current thread will be used as the platform thread. Ensure that the message loop is
   // initialized.
   fml::MessageLoop::EnsureInitializedForCurrentThread();
@@ -441,7 +442,8 @@ - (BOOL)createShell:(NSString*)entrypoint libraryURI:(NSString*)libraryURI {
   // synchronous.
   flutter::Shell::CreateCallback<flutter::PlatformView> on_create_platform_view =
       [](flutter::Shell& shell) {
-        return std::make_unique<flutter::PlatformViewIOS>(shell, shell.GetTaskRunners());
+        return std::make_unique<flutter::PlatformViewIOS>(
+            shell, flutter::GetRenderingAPIForProcess(), shell.GetTaskRunners());
       };
 
   flutter::Shell::CreateCallback<flutter::Rasterizer> on_create_rasterizer =

diff --git a/shell/platform/darwin/ios/framework/Source/FlutterOverlayView.h b/shell/platform/darwin/ios/framework/Source/FlutterOverlayView.h
@@ -13,7 +13,7 @@
 
 #include "flutter/fml/memory/weak_ptr.h"
 #include "flutter/shell/common/shell.h"
-#include "flutter/shell/platform/darwin/ios/ios_gl_context.h"
+#include "flutter/shell/platform/darwin/ios/ios_context_gl.h"
 #include "flutter/shell/platform/darwin/ios/ios_surface.h"
 #include "flutter/shell/platform/darwin/ios/ios_surface_gl.h"
 
@@ -36,7 +36,7 @@
 - (instancetype)init NS_DESIGNATED_INITIALIZER;
 - (instancetype)initWithContentsScale:(CGFloat)contentsScale;
 - (std::unique_ptr<flutter::IOSSurface>)createSurface:
-    (std::shared_ptr<flutter::IOSGLContext>)gl_context;
+    (std::shared_ptr<flutter::IOSContext>)ios_context;
 
 @end
 

diff --git a/shell/platform/darwin/ios/framework/Source/FlutterOverlayView.mm b/shell/platform/darwin/ios/framework/Source/FlutterOverlayView.mm
@@ -14,9 +14,6 @@
 #include "flutter/shell/common/rasterizer.h"
 #include "flutter/shell/platform/darwin/ios/framework/Source/FlutterView.h"
 #include "flutter/shell/platform/darwin/ios/ios_surface_gl.h"
-#if FLUTTER_SHELL_ENABLE_METAL
-#include "flutter/shell/platform/darwin/ios/ios_surface_metal.h"
-#endif
 #include "flutter/shell/platform/darwin/ios/ios_surface_software.h"
 #include "third_party/skia/include/utils/mac/SkCGUtils.h"
 
@@ -50,18 +47,11 @@ - (instancetype)init {
 - (instancetype)initWithContentsScale:(CGFloat)contentsScale {
   self = [self init];
 
-  if ([self.layer isKindOfClass:[CAEAGLLayer class]]) {
-    CAEAGLLayer* layer = reinterpret_cast<CAEAGLLayer*>(self.layer);
-    layer.allowsGroupOpacity = NO;
-    layer.contentsScale = contentsScale;
-    layer.rasterizationScale = contentsScale;
-#if FLUTTER_SHELL_ENABLE_METAL
-  } else if ([self.layer isKindOfClass:[CAMetalLayer class]]) {
-    CAMetalLayer* layer = reinterpret_cast<CAMetalLayer*>(self.layer);
-    layer.allowsGroupOpacity = NO;
-    layer.contentsScale = contentsScale;
-    layer.rasterizationScale = contentsScale;
-#endif  // FLUTTER_SHELL_ENABLE_METAL
+  if ([self.layer isKindOfClass:NSClassFromString(@"CAEAGLLayer")] ||
+      [self.layer isKindOfClass:NSClassFromString(@"CAMetalLayer")]) {
+    self.layer.allowsGroupOpacity = NO;
+    self.layer.contentsScale = contentsScale;
+    self.layer.rasterizationScale = contentsScale;
   }
 
   return self;
@@ -72,27 +62,11 @@ + (Class)layerClass {
 }
 
 - (std::unique_ptr<flutter::IOSSurface>)createSurface:
-    (std::shared_ptr<flutter::IOSGLContext>)gl_context {
-  if ([self.layer isKindOfClass:[CAEAGLLayer class]]) {
-    fml::scoped_nsobject<CAEAGLLayer> eagl_layer(
-        reinterpret_cast<CAEAGLLayer*>([self.layer retain]));
-    if (@available(iOS 9.0, *)) {
-      eagl_layer.get().presentsWithTransaction = YES;
-    }
-    return std::make_unique<flutter::IOSSurfaceGL>(std::move(eagl_layer), gl_context);
-#if FLUTTER_SHELL_ENABLE_METAL
-  } else if ([self.layer isKindOfClass:[CAMetalLayer class]]) {
-    fml::scoped_nsobject<CAMetalLayer> metalLayer(
-        reinterpret_cast<CAMetalLayer*>([self.layer retain]));
-    if (@available(iOS 8.0, *)) {
-      metalLayer.get().presentsWithTransaction = YES;
-    }
-    return std::make_unique<flutter::IOSSurfaceMetal>(std::move(metalLayer));
-#endif  //  FLUTTER_SHELL_ENABLE_METAL
-  } else {
-    fml::scoped_nsobject<CALayer> layer(reinterpret_cast<CALayer*>([self.layer retain]));
-    return std::make_unique<flutter::IOSSurfaceSoftware>(std::move(layer), nullptr);
-  }
+    (std::shared_ptr<flutter::IOSContext>)ios_context {
+  return flutter::IOSSurface::Create(std::move(ios_context),                              // context
+                                     fml::scoped_nsobject<CALayer>{[self.layer retain]},  // layer
+                                     nullptr  // platform views controller
+  );
 }
 
 // TODO(amirh): implement drawLayer to support snapshotting.