Add an API in ui_web to create a ui.Image from an ImageBitmap (f…

…lutter#45256) This API will help with situations in which the user has a browser resource that they want to transform into a `ui.Image` and render directly into the layer tree. Most browser resources can be converted to an `ImageBitmap` via the `createImageBitmap` API.
izouxv · Aug 30, 2023 · aeab1b0 · aeab1b0
1 parent b63eee2
commit aeab1b0
Show file tree

Hide file tree

Showing 19 changed files with 220 additions and 70 deletions.
diff --git a/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart b/lib/web_ui/lib/src/engine/canvaskit/canvaskit_api.dart
@@ -219,14 +219,31 @@ extension CanvasKitExtension on CanvasKit {
   ) => _MakeImage(info, pixels.toJS, bytesPerRow.toJS);
 
   @JS('MakeLazyImageFromTextureSource')
-  external SkImage? _MakeLazyImageFromTextureSource(
+  external SkImage? _MakeLazyImageFromTextureSource2(
     JSAny src,
     SkPartialImageInfo info,
   );
-  SkImage? MakeLazyImageFromTextureSource(
+
+  @JS('MakeLazyImageFromTextureSource')
+  external SkImage? _MakeLazyImageFromTextureSource3(
+    JSAny src,
+    JSNumber zeroSecondArgument,
+    JSBoolean srcIsPremultiplied,
+  );
+
+  SkImage? MakeLazyImageFromTextureSourceWithInfo(
     Object src,
     SkPartialImageInfo info,
-  ) => _MakeLazyImageFromTextureSource(src.toJSAnyShallow, info);
+  ) => _MakeLazyImageFromTextureSource2(src.toJSAnyShallow, info);
+
+  SkImage? MakeLazyImageFromImageBitmap(
+    DomImageBitmap imageBitmap,
+    bool hasPremultipliedAlpha,
+  ) => _MakeLazyImageFromTextureSource3(
+    imageBitmap as JSAny,
+    0.toJS,
+    hasPremultipliedAlpha.toJS,
+  );
 }
 
 @JS('window.CanvasKitInit')

diff --git a/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart b/lib/web_ui/lib/src/engine/canvaskit/image_web_codecs.dart
@@ -60,7 +60,7 @@ class CkBrowserImageDecoder extends BrowserImageDecoder {
 
   @override
   ui.Image generateImageFromVideoFrame(VideoFrame frame) {
-    final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSource(
+    final SkImage? skImage = canvasKit.MakeLazyImageFromTextureSourceWithInfo(
       frame,
       SkPartialImageInfo(
         alphaType: canvasKit.AlphaType.Premul,

diff --git a/lib/web_ui/lib/src/engine/canvaskit/renderer.dart b/lib/web_ui/lib/src/engine/canvaskit/renderer.dart
@@ -212,6 +212,18 @@ class CanvasKitRenderer implements Renderer {
     ui_web.ImageCodecChunkCallback? chunkCallback
   }) => skiaInstantiateWebImageCodec(uri.toString(), chunkCallback);
 
+  @override
+  ui.Image createImageFromImageBitmap(DomImageBitmap imageBitmap) {
+    final SkImage? skImage = canvasKit.MakeLazyImageFromImageBitmap(
+      imageBitmap,
+      true
+    );
+    if (skImage == null) {
+      throw Exception('Failed to convert image bitmap to an SkImage.');
+    }
+    return CkImage(skImage);
+  }
+
   @override
   void decodeImageFromPixels(
     Uint8List pixels,

diff --git a/lib/web_ui/lib/src/engine/dom.dart b/lib/web_ui/lib/src/engine/dom.dart
@@ -1433,6 +1433,33 @@ extension DomImageBitmapExtension on DomImageBitmap {
   external void close();
 }
 
+
+@JS('createImageBitmap')
+external JSPromise _createImageBitmap1(
+  JSAny source,
+);
+@JS('createImageBitmap')
+external JSPromise _createImageBitmap2(
+  JSAny source,
+  JSNumber x,
+  JSNumber y,
+  JSNumber width,
+  JSNumber height,
+);
+JSPromise createImageBitmap(JSAny source, [({int x, int y, int width, int height})? bounds]) {
+  if (bounds != null) {
+    return _createImageBitmap2(
+      source,
+      bounds.x.toJS,
+      bounds.y.toJS,
+      bounds.width.toJS,
+      bounds.height.toJS
+    );
+  } else {
+    return _createImageBitmap1(source);
+  }
+}
+
 @JS()
 @staticInterop
 class DomCanvasPattern {}
@@ -2264,14 +2291,24 @@ extension DomURLExtension on DomURL {
 @staticInterop
 class DomBlob {
   external factory DomBlob(JSArray parts);
+
+  external factory DomBlob.withOptions(JSArray parts, JSAny options);
 }
 
 extension DomBlobExtension on DomBlob {
   external JSPromise arrayBuffer();
 }
 
-DomBlob createDomBlob(List<Object?> parts) =>
-    DomBlob(parts.toJSAnyShallow as JSArray);
+DomBlob createDomBlob(List<Object?> parts, [Map<String, dynamic>? options]) {
+  if (options == null) {
+    return DomBlob(parts.toJSAnyShallow as JSArray);
+  } else {
+    return DomBlob.withOptions(
+      parts.toJSAnyShallow as JSArray,
+      options.toJSAnyDeep
+    );
+  }
+}
 
 typedef DomMutationCallback = void Function(
     JSArray mutation, DomMutationObserver observer);

diff --git a/lib/web_ui/lib/src/engine/html/renderer.dart b/lib/web_ui/lib/src/engine/html/renderer.dart
@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:js_interop';
 import 'dart:math' as math;
 import 'dart:typed_data';
 
@@ -361,4 +362,31 @@ class HtmlRenderer implements Renderer {
     baseline: baseline,
     lineNumber: lineNumber
   );
+
+  @override
+  Future<ui.Image> createImageFromImageBitmap(DomImageBitmap imageSource) async {
+    final int width = imageSource.width.toDartInt;
+    final int height = imageSource.height.toDartInt;
+    final OffScreenCanvas canvas = OffScreenCanvas(width, height);
+    final DomCanvasRenderingContextBitmapRenderer context = canvas.getBitmapRendererContext()!;
+    context.transferFromImageBitmap(imageSource);
+    final DomHTMLImageElement imageElement = createDomHTMLImageElement();
+    late final DomEventListener loadListener;
+    late final DomEventListener errorListener;
+    final Completer<HtmlImage> completer = Completer<HtmlImage>();
+    loadListener = createDomEventListener((DomEvent event) {
+      completer.complete(HtmlImage(imageElement, width, height));
+      imageElement.removeEventListener('load', loadListener);
+      imageElement.removeEventListener('error', errorListener);
+    });
+    errorListener = createDomEventListener((DomEvent event) {
+      completer.completeError(Exception('Failed to create image from image bitmap.'));
+      imageElement.removeEventListener('load', loadListener);
+      imageElement.removeEventListener('error', errorListener);
+    });
+    imageElement.addEventListener('load', loadListener);
+    imageElement.addEventListener('error', errorListener);
+    imageElement.src = await canvas.toDataUrl();
+    return completer.future;
+  }
 }
diff --git a/lib/web_ui/lib/src/engine/renderer.dart b/lib/web_ui/lib/src/engine/renderer.dart
@@ -6,17 +6,11 @@ import 'dart:async';
 import 'dart:math' as math;
 import 'dart:typed_data';
 
+import 'package:ui/src/engine.dart';
 import 'package:ui/src/engine/skwasm/skwasm_stub.dart' if (dart.library.ffi) 'package:ui/src/engine/skwasm/skwasm_impl.dart';
 import 'package:ui/ui.dart' as ui;
 import 'package:ui/ui_web/src/ui_web.dart' as ui_web;
 
-import 'browser_detection.dart';
-import 'canvaskit/renderer.dart';
-import 'configuration.dart';
-import 'embedder.dart';
-import 'fonts.dart';
-import 'html/renderer.dart';
-
 final Renderer _renderer = Renderer._internal();
 Renderer get renderer => _renderer;
 
@@ -134,6 +128,8 @@ abstract class Renderer {
     ui_web.ImageCodecChunkCallback? chunkCallback,
   });
 
+  FutureOr<ui.Image> createImageFromImageBitmap(DomImageBitmap imageSource);
+
   void decodeImageFromPixels(
     Uint8List pixels,
     int width,

diff --git a/lib/web_ui/lib/src/engine/safe_browser_api.dart b/lib/web_ui/lib/src/engine/safe_browser_api.dart
@@ -1003,6 +1003,12 @@ class OffScreenCanvas {
         : canvasElement!.getContext('2d');
   }
 
+  DomCanvasRenderingContextBitmapRenderer? getBitmapRendererContext() {
+    return (offScreenCanvas != null
+        ? offScreenCanvas!.getContext('bitmaprenderer')
+        : canvasElement!.getContext('bitmaprenderer')) as DomCanvasRenderingContextBitmapRenderer?;
+  }
+
   /// Feature detection for transferToImageBitmap on OffscreenCanvas.
   bool get transferToImageBitmapSupported =>
       js_util.hasProperty(offScreenCanvas!, 'transferToImageBitmap');

diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/codecs.dart
@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:js_interop';
+
 import 'package:ui/src/engine.dart';
 import 'package:ui/src/engine/skwasm/skwasm_impl.dart';
 import 'package:ui/ui.dart' as ui;
@@ -18,8 +20,8 @@ class SkwasmImageDecoder extends BrowserImageDecoder {
     final int width = frame.codedWidth.toInt();
     final int height = frame.codedHeight.toInt();
     final SkwasmSurface surface = (renderer as SkwasmRenderer).surface;
-    return SkwasmImage(imageCreateFromVideoFrame(
-      frame,
+    return SkwasmImage(imageCreateFromTextureSource(
+      frame as JSAny,
       width,
       height,
       surface.handle,

diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/raw/raw_image.dart
@@ -8,7 +8,6 @@ library skwasm_impl;
 import 'dart:ffi';
 import 'dart:js_interop';
 
-import 'package:ui/src/engine.dart';
 import 'package:ui/src/engine/skwasm/skwasm_impl.dart';
 
 final class RawImage extends Opaque {}
@@ -47,9 +46,9 @@ external ImageHandle imageCreateFromPixels(
 //   Int,
 //   Int,
 //   SurfaceHandle,
-// )>(symbol: 'image_createFromVideoFrame', isLeaf: true)
-// external ImageHandle imageCreateFromVideoFrame(
-//   JSAny videoFrame,
+// )>(symbol: 'image_createFromTextureSource', isLeaf: true)
+// external ImageHandle imageCreateFromTextureSource(
+//   JSAny textureSource,
 //   int width,
 //   int height,
 //   SurfaceHandle handle,
@@ -59,21 +58,21 @@ external ImageHandle imageCreateFromPixels(
 // annotations currently. For now, we can use JS interop to expose this function
 // instead.
 extension SkwasmImageExtension on SkwasmInstance {
-  @JS('wasmExports.image_createFromVideoFrame')
-  external JSNumber imageCreateFromVideoFrame(
-    VideoFrame frame,
+  @JS('wasmExports.image_createFromTextureSource')
+  external JSNumber imageCreateFromTextureSource(
+    JSAny textureSource,
     JSNumber width,
     JSNumber height,
     JSNumber surfaceHandle,
   );
 }
-ImageHandle imageCreateFromVideoFrame(
-  VideoFrame frame,
+ImageHandle imageCreateFromTextureSource(
+  JSAny frame,
   int width,
   int height,
   SurfaceHandle handle
 ) => ImageHandle.fromAddress(
-  skwasmInstance.imageCreateFromVideoFrame(
+  skwasmInstance.imageCreateFromTextureSource(
     frame,
     width.toJS,
     height.toJS,

diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_impl/renderer.dart
@@ -448,6 +448,16 @@ class SkwasmRenderer implements Renderer {
     baseline: baseline,
     lineNumber: lineNumber
   );
+
+  @override
+  ui.Image createImageFromImageBitmap(DomImageBitmap imageSource) {
+    return SkwasmImage(imageCreateFromTextureSource(
+      imageSource as JSAny,
+      imageSource.width.toDartInt,
+      imageSource.height.toDartInt,
+      surface.handle,
+    ));
+  }
 }
 
 class SkwasmPictureRenderer implements PictureRenderer {

diff --git a/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart b/lib/web_ui/lib/src/engine/skwasm/skwasm_stub/renderer.dart
@@ -189,4 +189,9 @@ class SkwasmRenderer implements Renderer {
     required double baseline,
     required int lineNumber
   }) => throw UnimplementedError('Skwasm not implemented on this platform.');
+
+  @override
+  ui.Image createImageFromImageBitmap(DomImageBitmap imageSource) {
+    throw UnimplementedError('Skwasm not implemented on this platform.');
+  }
 }
diff --git a/lib/web_ui/lib/ui_web/src/ui_web/images.dart b/lib/web_ui/lib/ui_web/src/ui_web/images.dart
@@ -2,6 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:js_interop';
+
 import 'package:ui/src/engine.dart';
 import 'package:ui/ui.dart' as ui;
 
@@ -25,3 +28,19 @@ Future<ui.Codec> createImageCodecFromUrl(
     chunkCallback: chunkCallback,
   );
 }
+
+/// Creates a [ui.Image] from an ImageBitmap object.
+///
+/// The contents of the ImageBitmap must have a premultiplied alpha.
+/// The engine will take ownership of the ImageBitmap object and consume its
+/// contents.
+///
+/// See https://developer.mozilla.org/en-US/docs/Web/API/ImageBitmap
+FutureOr<ui.Image> createImageFromImageBitmap(JSAny imageSource) {
+  if (!domInstanceOfString(imageSource, 'ImageBitmap')) {
+    throw ArgumentError('Image source $imageSource is not an ImageBitmap.', 'imageSource');
+  }
+  return renderer.createImageFromImageBitmap(
+    imageSource as DomImageBitmap,
+  );
+}
diff --git a/lib/web_ui/skwasm/image.cpp b/lib/web_ui/skwasm/image.cpp
@@ -79,20 +79,22 @@ class ExternalWebGLTexture : public GrExternalTexture {
 };
 }  // namespace
 
-class VideoFrameImageGenerator : public GrExternalTextureGenerator {
+class TextureSourceImageGenerator : public GrExternalTextureGenerator {
  public:
-  VideoFrameImageGenerator(SkImageInfo ii,
-                           SkwasmObject videoFrame,
-                           Skwasm::Surface* surface)
+  TextureSourceImageGenerator(SkImageInfo ii,
+                              SkwasmObject textureSource,
+                              Skwasm::Surface* surface)
       : GrExternalTextureGenerator(ii),
-        _videoFrameWrapper(surface->createVideoFrameWrapper(videoFrame)) {}
+        _textureSourceWrapper(
+            surface->createTextureSourceWrapper(textureSource)) {}
 
   std::unique_ptr<GrExternalTexture> generateExternalTexture(
       GrRecordingContext* context,
       GrMipMapped mipmapped) override {
     GrGLTextureInfo glInfo;
-    glInfo.fID = skwasm_createGlTextureFromVideoFrame(
-        _videoFrameWrapper->getVideoFrame(), fInfo.width(), fInfo.height());
+    glInfo.fID = skwasm_createGlTextureFromTextureSource(
+        _textureSourceWrapper->getTextureSource(), fInfo.width(),
+        fInfo.height());
     glInfo.fFormat = GL_RGBA8_OES;
     glInfo.fTarget = GL_TEXTURE_2D;
 
@@ -103,7 +105,7 @@ class VideoFrameImageGenerator : public GrExternalTextureGenerator {
   }
 
  private:
-  std::unique_ptr<Skwasm::VideoFrameWrapper> _videoFrameWrapper;
+  std::unique_ptr<Skwasm::TextureSourceWrapper> _textureSourceWrapper;
 };
 
 SKWASM_EXPORT SkImage* image_createFromPicture(SkPicture* picture,
@@ -129,17 +131,17 @@ SKWASM_EXPORT SkImage* image_createFromPixels(SkData* data,
       .release();
 }
 
-SKWASM_EXPORT SkImage* image_createFromVideoFrame(SkwasmObject videoFrame,
-                                                  int width,
-                                                  int height,
-                                                  Skwasm::Surface* surface) {
+SKWASM_EXPORT SkImage* image_createFromTextureSource(SkwasmObject textureSource,
+                                                     int width,
+                                                     int height,
+                                                     Skwasm::Surface* surface) {
   return SkImages::DeferredFromTextureGenerator(
-             std::unique_ptr<VideoFrameImageGenerator>(
-                 new VideoFrameImageGenerator(
+             std::unique_ptr<TextureSourceImageGenerator>(
+                 new TextureSourceImageGenerator(
                      SkImageInfo::Make(width, height,
                                        SkColorType::kRGBA_8888_SkColorType,
                                        SkAlphaType::kPremul_SkAlphaType),
-                     videoFrame, surface)))
+                     textureSource, surface)))
       .release();
 }