[Impeller] External OpenGLES texture sampling. (flutter#44559)

Allows for sampling from external textures in the OpenGLES backend.
DrawImageRect should Just Work if external textures are wrapped with the
new `kTextureExternalOES` type.

ci/licenses_golden/licenses_flutter

@@ -1082,6 +1082,8 @@ ORIGIN: ../../../flutter/impeller/compiler/compiler_backend.cc + ../../../flutte
 ORIGIN: ../../../flutter/impeller/compiler/compiler_backend.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/compiler_test.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/compiler_test.h + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/compiler/constants.cc + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/compiler/constants.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/impellerc_main.cc + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/include_dir.h + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/compiler/includer.cc + ../../../flutter/LICENSE
@@ -1348,6 +1350,7 @@ ORIGIN: ../../../flutter/impeller/entity/shaders/sweep_gradient_fill.frag + ../.
 ORIGIN: ../../../flutter/impeller/entity/shaders/sweep_gradient_ssbo_fill.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/entity/shaders/texture_fill.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/entity/shaders/texture_fill.vert + ../../../flutter/LICENSE
+ORIGIN: ../../../flutter/impeller/entity/shaders/texture_fill_external.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/entity/shaders/vertices.frag + ../../../flutter/LICENSE
 ORIGIN: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.frag + ../../../flutter/LICENSE
@@ -3797,6 +3800,8 @@ FILE: ../../../flutter/impeller/compiler/compiler_backend.cc
 FILE: ../../../flutter/impeller/compiler/compiler_backend.h
 FILE: ../../../flutter/impeller/compiler/compiler_test.cc
 FILE: ../../../flutter/impeller/compiler/compiler_test.h
+FILE: ../../../flutter/impeller/compiler/constants.cc
+FILE: ../../../flutter/impeller/compiler/constants.h
 FILE: ../../../flutter/impeller/compiler/impellerc_main.cc
 FILE: ../../../flutter/impeller/compiler/include_dir.h
 FILE: ../../../flutter/impeller/compiler/includer.cc
@@ -4063,6 +4068,7 @@ FILE: ../../../flutter/impeller/entity/shaders/sweep_gradient_fill.frag
 FILE: ../../../flutter/impeller/entity/shaders/sweep_gradient_ssbo_fill.frag
 FILE: ../../../flutter/impeller/entity/shaders/texture_fill.frag
 FILE: ../../../flutter/impeller/entity/shaders/texture_fill.vert
+FILE: ../../../flutter/impeller/entity/shaders/texture_fill_external.frag
 FILE: ../../../flutter/impeller/entity/shaders/tiled_texture_fill.frag
 FILE: ../../../flutter/impeller/entity/shaders/vertices.frag
 FILE: ../../../flutter/impeller/entity/shaders/yuv_to_rgb_filter.frag

impeller/compiler/BUILD.gn

@@ -36,6 +36,8 @@ impeller_component("compiler_lib") {
     "compiler.h",
     "compiler_backend.cc",
     "compiler_backend.h",
+    "constants.cc",
+    "constants.h",
     "include_dir.h",
     "includer.cc",
     "includer.h",

impeller/compiler/compiler.cc

@@ -14,11 +14,12 @@
 #include "flutter/fml/paths.h"
 #include "impeller/base/allocation.h"
 #include "impeller/compiler/compiler_backend.h"
+#include "impeller/compiler/constants.h"
 #include "impeller/compiler/includer.h"
 #include "impeller/compiler/logger.h"
 #include "impeller/compiler/spirv_compiler.h"
-#include "impeller/compiler/types.h"
 #include "impeller/compiler/uniform_sorter.h"
+#include "impeller/compiler/utilities.h"
 
 namespace impeller {
 namespace compiler {
@@ -128,6 +129,23 @@ static CompilerBackend CreateVulkanCompiler(
 static CompilerBackend CreateGLSLCompiler(const spirv_cross::ParsedIR& ir,
                                           const SourceOptions& source_options) {
   auto gl_compiler = std::make_shared<spirv_cross::CompilerGLSL>(ir);
+
+  // Walk the variables and insert the external image extension if any of them
+  // begins with the external texture prefix. Unfortunately, we can't walk
+  // `gl_compiler->get_shader_resources().separate_samplers` until the compiler
+  // is further along.
+  //
+  // Unfortunately, we can't just let the shader author add this extension and
+  // use `samplerExternalOES` directly because compiling to spirv requires the
+  // source language profile to be at least 310 ES, but this extension is
+  // incompatible with ES 310+.
+  for (auto& id : ir.ids_for_constant_or_variable) {
+    if (StringStartsWith(ir.get_name(id), kExternalTexturePrefix)) {
+      gl_compiler->require_extension("GL_OES_EGL_image_external");
+      break;
+    }
+  }
+
   spirv_cross::CompilerGLSL::Options sl_options;
   sl_options.force_zero_initialized_variables = true;
   sl_options.vertex.fixup_clipspace = true;
@@ -137,6 +155,13 @@ static CompilerBackend CreateGLSLCompiler(const spirv_cross::ParsedIR& ir,
                              ? source_options.gles_language_version
                              : 100;
     sl_options.es = true;
+    gl_compiler->set_variable_type_remap_callback(
+        [&](const spirv_cross::SPIRType& type, const std::string& var_name,
+            std::string& name_of_type) {
+          if (StringStartsWith(var_name, kExternalTexturePrefix)) {
+            name_of_type = "samplerExternalOES";
+          }
+        });
   } else {
     sl_options.version = source_options.gles_language_version > 0
                              ? source_options.gles_language_version

impeller/compiler/constants.cc

@@ -0,0 +1,13 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#include "impeller/compiler/constants.h"
+
+namespace impeller {
+namespace compiler {
+
+//
+
+}  // namespace compiler
+}  // namespace impeller

impeller/compiler/constants.h

@@ -0,0 +1,13 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+#pragma once
+
+namespace impeller {
+namespace compiler {
+
+constexpr char kExternalTexturePrefix[] = "SAMPLER_EXTERNAL_OES_";
+
+}  // namespace compiler
+}  // namespace impeller

impeller/compiler/reflector.cc

@@ -16,6 +16,7 @@
 #include "impeller/base/strings.h"
 #include "impeller/base/validation.h"
 #include "impeller/compiler/code_gen_template.h"
+#include "impeller/compiler/types.h"
 #include "impeller/compiler/uniform_sorter.h"
 #include "impeller/compiler/utilities.h"
 #include "impeller/geometry/half.h"

impeller/compiler/utilities.cc

@@ -61,5 +61,17 @@ std::string ConvertToEntrypointName(std::string_view string) {
   return stream.str();
 }
 
+bool StringStartsWith(const std::string& target, const std::string& prefix) {
+  if (prefix.length() > target.length()) {
+    return false;
+  }
+  for (size_t i = 0; i < prefix.length(); i++) {
+    if (target[i] != prefix[i]) {
+      return false;
+    }
+  }
+  return true;
+}
+
 }  // namespace compiler
 }  // namespace impeller

impeller/compiler/utilities.h

@@ -29,5 +29,7 @@ std::string ConvertToCamelCase(std::string_view string);
 ///         language.
 std::string ConvertToEntrypointName(std::string_view string);
 
+bool StringStartsWith(const std::string& target, const std::string& prefix);
+
 }  // namespace compiler
 }  // namespace impeller

impeller/core/formats.h

@@ -234,6 +234,7 @@ enum class TextureType {
   kTexture2D,
   kTexture2DMultisample,
   kTextureCube,
+  kTextureExternalOES,
 };
 
 constexpr const char* TextureTypeToString(TextureType type) {
@@ -244,6 +245,8 @@ constexpr const char* TextureTypeToString(TextureType type) {
       return "Texture2DMultisample";
     case TextureType::kTextureCube:
       return "TextureCube";
+    case TextureType::kTextureExternalOES:
+      return "TextureExternalOES";
   }
   FML_UNREACHABLE();
 }
@@ -252,6 +255,7 @@ constexpr bool IsMultisampleCapable(TextureType type) {
   switch (type) {
     case TextureType::kTexture2D:
     case TextureType::kTextureCube:
+    case TextureType::kTextureExternalOES:
       return false;
     case TextureType::kTexture2DMultisample:
       return true;

impeller/core/texture.cc

@@ -62,6 +62,7 @@ bool Texture::IsSliceValid(size_t slice) const {
   switch (desc_.type) {
     case TextureType::kTexture2D:
     case TextureType::kTexture2DMultisample:
+    case TextureType::kTextureExternalOES:
       return slice == 0;
     case TextureType::kTextureCube:
       return slice <= 5;

impeller/entity/BUILD.gn

@@ -65,6 +65,7 @@ impeller_shaders("entity_shaders") {
     "shaders/sweep_gradient_fill.frag",
     "shaders/texture_fill.frag",
     "shaders/texture_fill.vert",
+    "shaders/texture_fill_external.frag",
     "shaders/tiled_texture_fill.frag",
     "shaders/vertices.frag",
     "shaders/yuv_to_rgb_filter.frag",

impeller/entity/contents/content_context.h

@@ -8,6 +8,7 @@
 #include <optional>
 #include <unordered_map>
 
+#include "flutter/fml/build_config.h"
 #include "flutter/fml/hash_combine.h"
 #include "flutter/fml/logging.h"
 #include "flutter/fml/macros.h"
@@ -53,6 +54,7 @@
 #include "impeller/entity/sweep_gradient_fill.frag.h"
 #include "impeller/entity/texture_fill.frag.h"
 #include "impeller/entity/texture_fill.vert.h"
+#include "impeller/entity/texture_fill_external.frag.h"
 #include "impeller/entity/tiled_texture_fill.frag.h"
 #include "impeller/entity/uv.comp.h"
 #include "impeller/entity/vertices.frag.h"
@@ -143,6 +145,8 @@ using RRectBlurPipeline =
 using BlendPipeline = RenderPipelineT<BlendVertexShader, BlendFragmentShader>;
 using TexturePipeline =
     RenderPipelineT<TextureFillVertexShader, TextureFillFragmentShader>;
+using TextureExternalPipeline =
+    RenderPipelineT<TextureFillVertexShader, TextureFillExternalFragmentShader>;
 using PositionUVPipeline =
     RenderPipelineT<TextureFillVertexShader, TiledTextureFillFragmentShader>;
 using TiledTexturePipeline =
@@ -413,6 +417,11 @@ class ContentContext {
     return GetPipeline(texture_pipelines_, opts);
   }
 
+  std::shared_ptr<Pipeline<PipelineDescriptor>> GetTextureExternalPipeline(
+      ContentContextOptions opts) const {
+    return GetPipeline(texture_external_pipelines_, opts);
+  }
+
   std::shared_ptr<Pipeline<PipelineDescriptor>> GetPositionUVPipeline(
       ContentContextOptions opts) const {
     return GetPipeline(position_uv_pipelines_, opts);
@@ -733,6 +742,7 @@ class ContentContext {
   mutable Variants<RRectBlurPipeline> rrect_blur_pipelines_;
   mutable Variants<BlendPipeline> texture_blend_pipelines_;
   mutable Variants<TexturePipeline> texture_pipelines_;
+  mutable Variants<TextureExternalPipeline> texture_external_pipelines_;
   mutable Variants<PositionUVPipeline> position_uv_pipelines_;
   mutable Variants<TiledTexturePipeline> tiled_texture_pipelines_;
   mutable Variants<GaussianBlurAlphaDecalPipeline>

impeller/entity/contents/texture_contents.cc

@@ -149,7 +149,12 @@ bool TextureContents::Render(const ContentContext& renderer,
   }
   pipeline_options.primitive_type = PrimitiveType::kTriangleStrip;
 
-  cmd.pipeline = renderer.GetTexturePipeline(pipeline_options);
+  if (texture_->GetTextureDescriptor().type ==
+      TextureType::kTextureExternalOES) {
+    cmd.pipeline = renderer.GetTextureExternalPipeline(pipeline_options);
+  } else {
+    cmd.pipeline = renderer.GetTexturePipeline(pipeline_options);
+  }
   cmd.stencil_reference = entity.GetStencilDepth();
   cmd.BindVertices(vertex_builder.CreateVertexBuffer(host_buffer));
   VS::BindFrameInfo(cmd, host_buffer.EmplaceUniform(frame_info));

impeller/entity/shaders/texture_fill_external.frag

@@ -0,0 +1,16 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+uniform sampler2D SAMPLER_EXTERNAL_OES_texture_sampler;
+
+in vec2 v_texture_coords;
+in float v_alpha;
+
+out vec4 frag_color;
+
+void main() {
+  vec4 sampled =
+      texture(SAMPLER_EXTERNAL_OES_texture_sampler, v_texture_coords);
+  frag_color = sampled * v_alpha;
+}

impeller/playground/BUILD.gn

@@ -41,9 +41,6 @@ impeller_component("playground") {
   }
 
   public_deps = [
-    "../entity:entity_shaders",
-    "../entity:framebuffer_blend_entity_shaders",
-    "../entity:modern_entity_shaders",
     "../fixtures:shader_fixtures",
     "../image:image_skia_backend",
     "../renderer",
@@ -54,6 +51,14 @@ impeller_component("playground") {
     "//third_party/imgui:imgui_glfw",
   ]
 
+  if (impeller_supports_rendering) {
+    public_deps += [
+      "../entity:entity_shaders",
+      "../entity:framebuffer_blend_entity_shaders",
+      "../entity:modern_entity_shaders",
+    ]
+  }
+
   if (is_mac) {
     frameworks = [
       "AppKit.framework",

impeller/renderer/backend/gles/formats_gles.h

@@ -174,6 +174,8 @@ constexpr GLenum ToTextureType(TextureType type) {
       return GL_TEXTURE_2D_MULTISAMPLE;
     case TextureType::kTextureCube:
       return GL_TEXTURE_CUBE_MAP;
+    case TextureType::kTextureExternalOES:
+      return GL_TEXTURE_EXTERNAL_OES;
   }
   FML_UNREACHABLE();
 }
@@ -186,6 +188,8 @@ constexpr std::optional<GLenum> ToTextureTarget(TextureType type) {
       return std::nullopt;
     case TextureType::kTextureCube:
       return GL_TEXTURE_CUBE_MAP;
+    case TextureType::kTextureExternalOES:
+      return GL_TEXTURE_EXTERNAL_OES;
   }
   FML_UNREACHABLE();
 }

impeller/renderer/backend/gles/texture_gles.cc

@@ -246,6 +246,10 @@ bool TextureGLES::OnSetContents(std::shared_ptr<const fml::Mapping> mapping,
       texture_type = GL_TEXTURE_CUBE_MAP;
       texture_target = GL_TEXTURE_CUBE_MAP_POSITIVE_X + slice;
       break;
+    case TextureType::kTextureExternalOES:
+      texture_type = GL_TEXTURE_EXTERNAL_OES;
+      texture_target = GL_TEXTURE_EXTERNAL_OES;
+      break;
   }
 
   auto data = std::make_shared<TexImage2DData>(tex_descriptor.format,
@@ -440,6 +444,8 @@ bool TextureGLES::GenerateMipmap() {
       return false;
     case TextureType::kTextureCube:
       break;
+    case TextureType::kTextureExternalOES:
+      break;
   }
 
   if (!Bind()) {

impeller/renderer/backend/metal/formats_mtl.h

@@ -9,6 +9,7 @@
 #include <optional>
 
 #include "flutter/fml/macros.h"
+#include "impeller/base/validation.h"
 #include "impeller/core/formats.h"
 #include "impeller/core/texture_descriptor.h"
 #include "impeller/geometry/color.h"
@@ -367,6 +368,9 @@ constexpr MTLTextureType ToMTLTextureType(TextureType type) {
       return MTLTextureType2DMultisample;
     case TextureType::kTextureCube:
       return MTLTextureTypeCube;
+    case TextureType::kTextureExternalOES:
+      VALIDATION_LOG
+          << "kTextureExternalOES can not be used with the Metal backend.";
   }
   return MTLTextureType2D;
 }

impeller/renderer/backend/vulkan/formats_vk.h

@@ -5,6 +5,7 @@
 #pragma once
 
 #include "flutter/fml/macros.h"
+#include "impeller/base/validation.h"
 #include "impeller/core/formats.h"
 #include "impeller/core/shader_types.h"
 #include "impeller/renderer/backend/vulkan/vk.h"
@@ -587,6 +588,9 @@ constexpr uint32_t ToArrayLayerCount(TextureType type) {
       return 1u;
     case TextureType::kTextureCube:
       return 6u;
+    case TextureType::kTextureExternalOES:
+      VALIDATION_LOG
+          << "kTextureExternalOES can not be used with the Vulkan backend.";
   }
   FML_UNREACHABLE();
 }
@@ -598,6 +602,9 @@ constexpr vk::ImageViewType ToVKImageViewType(TextureType type) {
       return vk::ImageViewType::e2D;
     case TextureType::kTextureCube:
       return vk::ImageViewType::eCube;
+    case TextureType::kTextureExternalOES:
+      VALIDATION_LOG
+          << "kTextureExternalOES can not be used with the Vulkan backend.";
   }
   FML_UNREACHABLE();
 }
@@ -609,6 +616,9 @@ constexpr vk::ImageCreateFlags ToVKImageCreateFlags(TextureType type) {
       return {};
     case TextureType::kTextureCube:
       return vk::ImageCreateFlagBits::eCubeCompatible;
+    case TextureType::kTextureExternalOES:
+      VALIDATION_LOG
+          << "kTextureExternalOES can not be used with the Vulkan backend.";
   }
   FML_UNREACHABLE();
 }