Skip to content

Commit

Permalink
[embedder] [metal] Embedder API can support externally composited Met…
Browse files Browse the repository at this point in the history
…al textures (flutter#24327)
  • Loading branch information
iskakaushik authored Feb 18, 2021
1 parent 4956ad9 commit c902a16
Show file tree
Hide file tree
Showing 23 changed files with 796 additions and 206 deletions.
6 changes: 6 additions & 0 deletions ci/licenses_golden/licenses_flutter
Original file line number Diff line number Diff line change
Expand Up @@ -1099,6 +1099,7 @@ FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBacki
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterBackingStoreData.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterDartProject_Internal.h
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEmbedderExternalTextureUnittests.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngineTest.mm
FILE: ../../../flutter/shell/platform/darwin/macos/framework/Source/FlutterEngine_Internal.h
Expand Down Expand Up @@ -1152,6 +1153,10 @@ FILE: ../../../flutter/shell/platform/embedder/embedder_engine.cc
FILE: ../../../flutter/shell/platform/embedder/embedder_engine.h
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.cc
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_gl.h
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.h
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_metal.mm
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.cc
FILE: ../../../flutter/shell/platform/embedder/embedder_external_texture_resolver.h
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.cc
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view.h
FILE: ../../../flutter/shell/platform/embedder/embedder_external_view_embedder.cc
Expand Down Expand Up @@ -1187,6 +1192,7 @@ FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_platform
FILE: ../../../flutter/shell/platform/embedder/fixtures/compositor_with_root_layer_only.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_noxform.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/dpr_xform.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/external_texture_metal.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_metal.png
FILE: ../../../flutter/shell/platform/embedder/fixtures/gradient_xform.png
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,22 @@

#import "flutter/shell/platform/darwin/common/framework/Headers/FlutterTexture.h"
#include "third_party/skia/include/core/SkCanvas.h"
#include "third_party/skia/include/gpu/GrDirectContext.h"
#include "third_party/skia/include/core/SkImage.h"

@interface FlutterDarwinExternalTextureSkImageWrapper : NSObject

+ (sk_sp<SkImage>)wrapYUVATexture:(nonnull id<MTLTexture>)yTex
UVTex:(nonnull id<MTLTexture>)uvTex
grContext:(nonnull GrDirectContext*)grContext
width:(size_t)width
height:(size_t)height;

+ (sk_sp<SkImage>)wrapRGBATexture:(nonnull id<MTLTexture>)rgbaTex
grContext:(nonnull GrDirectContext*)grContext
width:(size_t)width
height:(size_t)height;

@end

@interface FlutterDarwinExternalTextureMetal : NSObject

Expand Down
109 changes: 63 additions & 46 deletions shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.mm
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,6 @@

FLUTTER_ASSERT_ARC

namespace {

static sk_cf_obj<const void*> SkiaTextureFromCVMetalTexture(CVMetalTextureRef cvMetalTexture) {
id<MTLTexture> texture = CVMetalTextureGetTexture(cvMetalTexture);
// CVMetal texture can be released as soon as we can the MTLTexture from it.
CVPixelBufferRelease(cvMetalTexture);
return sk_cf_obj<const void*>{(__bridge_retained const void*)texture};
}

}

@implementation FlutterDarwinExternalTextureMetal {
CVMetalTextureCacheRef _textureCache;
NSObject<FlutterTexture>* _externalTexture;
Expand Down Expand Up @@ -185,31 +174,17 @@ - (void)onTextureUnregistered {
}
}

GrMtlTextureInfo ySkiaTextureInfo;
ySkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(yMetalTexture);
id<MTLTexture> yTex = CVMetalTextureGetTexture(yMetalTexture);
CVBufferRelease(yMetalTexture);

GrBackendTexture skiaBackendTextures[2];
skiaBackendTextures[0] = GrBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/ySkiaTextureInfo);
id<MTLTexture> uvTex = CVMetalTextureGetTexture(uvMetalTexture);
CVBufferRelease(uvMetalTexture);

GrMtlTextureInfo uvSkiaTextureInfo;
uvSkiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(uvMetalTexture);

skiaBackendTextures[1] = GrBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/uvSkiaTextureInfo);
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
kTopLeft_GrSurfaceOrigin);

sk_sp<SkImage> image =
SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
return image;
return [FlutterDarwinExternalTextureSkImageWrapper wrapYUVATexture:yTex
UVTex:uvTex
grContext:grContext
width:textureSize.width()
height:textureSize.height()];
}

- (sk_sp<SkImage>)wrapRGBAExternalPixelBuffer:(CVPixelBufferRef)pixelBuffer
Expand All @@ -233,22 +208,64 @@ GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
return nullptr;
}

id<MTLTexture> rgbaTex = CVMetalTextureGetTexture(metalTexture);
CVBufferRelease(metalTexture);

return [FlutterDarwinExternalTextureSkImageWrapper wrapRGBATexture:rgbaTex
grContext:grContext
width:textureSize.width()
height:textureSize.height()];
}

@end

@implementation FlutterDarwinExternalTextureSkImageWrapper

+ (sk_sp<SkImage>)wrapYUVATexture:(id<MTLTexture>)yTex
UVTex:(id<MTLTexture>)uvTex
grContext:(nonnull GrDirectContext*)grContext
width:(size_t)width
height:(size_t)height {
GrMtlTextureInfo ySkiaTextureInfo;
ySkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)yTex};

GrBackendTexture skiaBackendTextures[2];
skiaBackendTextures[0] = GrBackendTexture(/*width=*/width,
/*height=*/height,
/*mipMapped=*/GrMipMapped::kNo,
/*textureInfo=*/ySkiaTextureInfo);

GrMtlTextureInfo uvSkiaTextureInfo;
uvSkiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)uvTex};

skiaBackendTextures[1] = GrBackendTexture(/*width=*/width,
/*height=*/height,
/*mipMapped=*/GrMipMapped::kNo,
/*textureInfo=*/uvSkiaTextureInfo);
SkYUVAInfo yuvaInfo(skiaBackendTextures[0].dimensions(), SkYUVAInfo::PlaneConfig::kY_UV,
SkYUVAInfo::Subsampling::k444, kRec601_SkYUVColorSpace);
GrYUVABackendTextures yuvaBackendTextures(yuvaInfo, skiaBackendTextures,
kTopLeft_GrSurfaceOrigin);

return SkImage::MakeFromYUVATextures(grContext, yuvaBackendTextures, /*imageColorSpace=*/nullptr,
/*releaseProc*/ nullptr, /*releaseContext*/ nullptr);
}

+ (sk_sp<SkImage>)wrapRGBATexture:(id<MTLTexture>)rgbaTex
grContext:(nonnull GrDirectContext*)grContext
width:(size_t)width
height:(size_t)height {
GrMtlTextureInfo skiaTextureInfo;
skiaTextureInfo.fTexture = SkiaTextureFromCVMetalTexture(metalTexture);
skiaTextureInfo.fTexture = sk_cf_obj<const void*>{(__bridge_retained const void*)rgbaTex};

GrBackendTexture skiaBackendTexture(/*width=*/textureSize.width(),
/*height=*/textureSize.height(),
GrBackendTexture skiaBackendTexture(/*width=*/width,
/*height=*/height,
/*mipMapped=*/GrMipMapped ::kNo,
/*textureInfo=*/skiaTextureInfo);

sk_sp<SkImage> image =
SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
/*releaseContext*/ nullptr

);
return image;
return SkImage::MakeFromTexture(grContext, skiaBackendTexture, kTopLeft_GrSurfaceOrigin,
kBGRA_8888_SkColorType, kPremul_SkAlphaType,
/*imageColorSpace=*/nullptr, /*releaseProc*/ nullptr,
/*releaseContext*/ nullptr);
}

@end
5 changes: 5 additions & 0 deletions shell/platform/darwin/macos/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,7 @@ executable("flutter_desktop_darwin_unittests") {

if (shell_enable_metal) {
sources += [
"framework/Source/FlutterEmbedderExternalTextureUnittests.mm",
"framework/Source/FlutterMetalRendererTest.mm",
"framework/Source/FlutterMetalSurfaceManagerTest.mm",
]
Expand All @@ -177,6 +178,10 @@ executable("flutter_desktop_darwin_unittests") {
"//flutter/testing:testing_lib",
"//third_party/ocmock:ocmock",
]

if (shell_enable_metal) {
deps += [ "//flutter/shell/platform/darwin/graphics" ]
}
}

copy("copy_dylib") {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#import <Foundation/Foundation.h>
#import <Metal/Metal.h>

#include <memory>
#include <vector>

#import "flutter/shell/platform/darwin/graphics/FlutterDarwinContextMetal.h"
#import "flutter/shell/platform/darwin/graphics/FlutterDarwinExternalTextureMetal.h"
#include "flutter/shell/platform/embedder/embedder.h"
#include "flutter/shell/platform/embedder/embedder_external_texture_metal.h"
#import "flutter/testing/testing.h"
#include "third_party/googletest/googletest/include/gtest/gtest.h"
#include "third_party/skia/include/core/SkImage.h"
#include "third_party/skia/include/core/SkSamplingOptions.h"
#include "third_party/skia/include/core/SkSurface.h"

namespace flutter::testing {

TEST(FlutterEmbedderExternalTextureUnittests, TestTextureResolution) {
// constants.
const size_t width = 100;
const size_t height = 100;
const int64_t texture_id = 1;

// setup the surface.
FlutterDarwinContextMetal* darwinContextMetal =
[[FlutterDarwinContextMetal alloc] initWithDefaultMTLDevice];
SkImageInfo info = SkImageInfo::MakeN32Premul(width, height);
GrDirectContext* grContext = darwinContextMetal.mainContext.get();
sk_sp<SkSurface> gpuSurface(SkSurface::MakeRenderTarget(grContext, SkBudgeted::kNo, info));

// create a texture.
MTLTextureDescriptor* textureDescriptor = [[MTLTextureDescriptor alloc] init];
textureDescriptor.pixelFormat = MTLPixelFormatBGRA8Unorm;
textureDescriptor.width = width;
textureDescriptor.height = height;
textureDescriptor.usage = MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead;
id<MTLTexture> mtlTexture =
[darwinContextMetal.device newTextureWithDescriptor:textureDescriptor];

// callback to resolve the texture.
EmbedderExternalTextureMetal::ExternalTextureCallback callback = [&](int64_t texture_id, size_t w,
size_t h) {
EXPECT_TRUE(w == width);
EXPECT_TRUE(h == height);

FlutterMetalExternalTexture* texture = new FlutterMetalExternalTexture();
texture->struct_size = sizeof(FlutterMetalExternalTexture);
texture->num_textures = 1;
texture->height = h;
texture->width = w;
texture->pixel_format = FlutterMetalExternalTexturePixelFormat::kRGBA;

std::vector<FlutterMetalTextureHandle> textures = {
(__bridge FlutterMetalTextureHandle)mtlTexture,
};

texture->textures = textures.data();

return std::unique_ptr<FlutterMetalExternalTexture>(texture);
};

// render the texture.
std::unique_ptr<flutter::Texture> texture =
std::make_unique<EmbedderExternalTextureMetal>(texture_id, callback);
SkRect bounds = SkRect::MakeWH(info.width(), info.height());
SkSamplingOptions sampling = SkSamplingOptions(SkFilterMode::kNearest);
texture->Paint(*gpuSurface->getCanvas(), bounds, /*freeze=*/false, grContext, sampling);

ASSERT_TRUE(mtlTexture != nil);

gpuSurface->makeImageSnapshot();
}

} // namespace flutter::testing
7 changes: 6 additions & 1 deletion shell/platform/embedder/BUILD.gn
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ template("embedder_source_set") {
"embedder.cc",
"embedder_engine.cc",
"embedder_engine.h",
"embedder_external_texture_resolver.cc",
"embedder_external_texture_resolver.h",
"embedder_external_view.cc",
"embedder_external_view.h",
"embedder_external_view_embedder.cc",
Expand Down Expand Up @@ -90,6 +92,8 @@ template("embedder_source_set") {

if (embedder_enable_metal) {
sources += [
"embedder_external_texture_metal.h",
"embedder_external_texture_metal.mm",
"embedder_surface_metal.h",
"embedder_surface_metal.mm",
]
Expand Down Expand Up @@ -156,6 +160,7 @@ test_fixtures("fixtures") {
"fixtures/dpr_xform.png",
"fixtures/gradient.png",
"fixtures/gradient_metal.png",
"fixtures/external_texture_metal.png",
"fixtures/gradient_xform.png",
"fixtures/scene_without_custom_compositor.png",
"fixtures/scene_without_custom_compositor_with_xform.png",
Expand Down Expand Up @@ -226,7 +231,7 @@ if (enable_unittests) {
sources += [
"tests/embedder_test_context_metal.cc",
"tests/embedder_test_context_metal.h",
"tests/embedder_unittests_metal.cc",
"tests/embedder_unittests_metal.mm",
]

deps += [ "//flutter/testing:metal" ]
Expand Down
Loading

0 comments on commit c902a16

Please sign in to comment.