Android: Add helper class for generating OpenGL shaders

This CL adds a helper class GlShaderBuilder to build an instances of RendererCommon.GlDrawer that can accept multiple input sources (OES, RGB, or YUV) using a generic fragment shader as input. Bug: webrtc:9355 Change-Id: I14a0a280d2b6f838984f7b60897cc0c58e2a948a Reviewed-on: https://webrtc-review.googlesource.com/80940 Commit-Queue: Magnus Jedvert <[email protected]> Reviewed-by: Sami Kalliomäki <[email protected]> Cr-Commit-Position: refs/heads/master@{#23622}
Yeyoon · Jun 15, 2018 · 65c61dc · 65c61dc
1 parent 8643b78
commit 65c61dc
Show file tree

Hide file tree

Showing 7 changed files with 574 additions and 416 deletions.
diff --git a/BUILD.gn b/BUILD.gn
@@ -584,6 +584,7 @@ if (rtc_include_tests) {
         "examples/androidjunit/src/org/appspot/apprtc/BluetoothManagerTest.java",
         "examples/androidjunit/src/org/appspot/apprtc/DirectRTCClientTest.java",
         "examples/androidjunit/src/org/appspot/apprtc/TCPChannelClientTest.java",
+        "sdk/android/tests/src/org/webrtc/GlGenericDrawerTest.java",
         "sdk/android/tests/src/org/webrtc/CameraEnumerationTest.java",
         "sdk/android/tests/src/org/webrtc/ScalingSettingsTest.java",
       ]

diff --git a/sdk/android/BUILD.gn b/sdk/android/BUILD.gn
@@ -757,6 +757,8 @@ rtc_android_library("video_api_java") {
     "api/org/webrtc/SurfaceTextureHelper.java",
     "api/org/webrtc/TextureBufferImpl.java",
     "api/org/webrtc/YuvConverter.java",
+    "api/org/webrtc/VideoFrameDrawer.java",
+    "src/java/org/webrtc/GlGenericDrawer.java",
     "api/org/webrtc/YuvHelper.java",
     "src/java/org/webrtc/EglBase10.java",
     "src/java/org/webrtc/EglBase14.java",
@@ -823,7 +825,6 @@ rtc_android_library("video_java") {
     "api/org/webrtc/GlRectDrawer.java",
     "api/org/webrtc/VideoDecoderFallback.java",
     "api/org/webrtc/VideoEncoderFallback.java",
-    "api/org/webrtc/VideoFrameDrawer.java",
     "src/java/org/webrtc/NativeCapturerObserver.java",
     "src/java/org/webrtc/NV21Buffer.java",
     "src/java/org/webrtc/VideoDecoderWrapper.java",

diff --git a/sdk/android/api/org/webrtc/GlRectDrawer.java b/sdk/android/api/org/webrtc/GlRectDrawer.java
@@ -10,201 +10,22 @@
 
 package org.webrtc;
 
-import android.opengl.GLES11Ext;
-import android.opengl.GLES20;
-import java.nio.FloatBuffer;
-import java.util.IdentityHashMap;
-import java.util.Map;
-
-/**
- * Helper class to draw an opaque quad on the target viewport location. Rotation, mirror, and
- * cropping is specified using a 4x4 texture coordinate transform matrix. The frame input can either
- * be an OES texture or YUV textures in I420 format. The GL state must be preserved between draw
- * calls, this is intentional to maximize performance. The function release() must be called
- * manually to free the resources held by this object.
- */
-public class GlRectDrawer implements RendererCommon.GlDrawer {
-  // clang-format off
-  // Simple vertex shader, used for both YUV and OES.
-  private static final String VERTEX_SHADER_STRING =
-        "varying vec2 interp_tc;\n"
-      + "attribute vec4 in_pos;\n"
-      + "attribute vec4 in_tc;\n"
-      + "\n"
-      + "uniform mat4 texMatrix;\n"
-      + "\n"
-      + "void main() {\n"
-      + "    gl_Position = in_pos;\n"
-      + "    interp_tc = (texMatrix * in_tc).xy;\n"
-      + "}\n";
-
-  private static final String YUV_FRAGMENT_SHADER_STRING =
-        "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "\n"
-      + "uniform sampler2D y_tex;\n"
-      + "uniform sampler2D u_tex;\n"
-      + "uniform sampler2D v_tex;\n"
-      + "\n"
-      + "void main() {\n"
-      // CSC according to http://www.fourcc.org/fccyvrgb.php
-      + "  float y = texture2D(y_tex, interp_tc).r;\n"
-      + "  float u = texture2D(u_tex, interp_tc).r - 0.5;\n"
-      + "  float v = texture2D(v_tex, interp_tc).r - 0.5;\n"
-      + "  gl_FragColor = vec4(y + 1.403 * v, "
-      + "                      y - 0.344 * u - 0.714 * v, "
-      + "                      y + 1.77 * u, 1);\n"
-      + "}\n";
-
-  private static final String RGB_FRAGMENT_SHADER_STRING =
-        "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "\n"
-      + "uniform sampler2D rgb_tex;\n"
-      + "\n"
-      + "void main() {\n"
-      + "  gl_FragColor = texture2D(rgb_tex, interp_tc);\n"
+/** Simplest possible GL shader that just draws frames as opaque quads. */
+public class GlRectDrawer extends GlGenericDrawer {
+  private static final String FRAGMENT_SHADER = "void main() {\n"
+      + "  gl_FragColor = sample(tc);\n"
       + "}\n";
 
-  private static final String OES_FRAGMENT_SHADER_STRING =
-        "#extension GL_OES_EGL_image_external : require\n"
-      + "precision mediump float;\n"
-      + "varying vec2 interp_tc;\n"
-      + "\n"
-      + "uniform samplerExternalOES oes_tex;\n"
-      + "\n"
-      + "void main() {\n"
-      + "  gl_FragColor = texture2D(oes_tex, interp_tc);\n"
-      + "}\n";
-  // clang-format on
-
-  // Vertex coordinates in Normalized Device Coordinates, i.e. (-1, -1) is bottom-left and (1, 1) is
-  // top-right.
-  private static final FloatBuffer FULL_RECTANGLE_BUF = GlUtil.createFloatBuffer(new float[] {
-      -1.0f, -1.0f, // Bottom left.
-      1.0f, -1.0f, // Bottom right.
-      -1.0f, 1.0f, // Top left.
-      1.0f, 1.0f, // Top right.
-  });
-
-  // Texture coordinates - (0, 0) is bottom-left and (1, 1) is top-right.
-  private static final FloatBuffer FULL_RECTANGLE_TEX_BUF = GlUtil.createFloatBuffer(new float[] {
-      0.0f, 0.0f, // Bottom left.
-      1.0f, 0.0f, // Bottom right.
-      0.0f, 1.0f, // Top left.
-      1.0f, 1.0f // Top right.
-  });
-
-  private static class Shader {
-    public final GlShader glShader;
-    public final int texMatrixLocation;
-
-    public Shader(String fragmentShader) {
-      this.glShader = new GlShader(VERTEX_SHADER_STRING, fragmentShader);
-      this.texMatrixLocation = glShader.getUniformLocation("texMatrix");
-    }
-  }
-
-  // The keys are one of the fragments shaders above.
-  private final Map<String, Shader> shaders = new IdentityHashMap<String, Shader>();
-
-  /**
-   * Draw an OES texture frame with specified texture transformation matrix. Required resources are
-   * allocated at the first call to this function.
-   */
-  @Override
-  public void drawOes(int oesTextureId, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(OES_FRAGMENT_SHADER_STRING, texMatrix);
-    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
-    // updateTexImage() may be called from another thread in another EGL context, so we need to
-    // bind/unbind the texture in each draw call so that GLES understads it's a new texture.
-    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, oesTextureId);
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, 0);
-  }
-
-  /**
-   * Draw a RGB(A) texture frame with specified texture transformation matrix. Required resources
-   * are allocated at the first call to this function.
-   */
-  @Override
-  public void drawRgb(int textureId, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(RGB_FRAGMENT_SHADER_STRING, texMatrix);
-    GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
-    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, textureId);
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    // Unbind the texture as a precaution.
-    GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
-  }
-
-  /**
-   * Draw a YUV frame with specified texture transformation matrix. Required resources are
-   * allocated at the first call to this function.
-   */
-  @Override
-  public void drawYuv(int[] yuvTextures, float[] texMatrix, int frameWidth, int frameHeight,
-      int viewportX, int viewportY, int viewportWidth, int viewportHeight) {
-    prepareShader(YUV_FRAGMENT_SHADER_STRING, texMatrix);
-    // Bind the textures.
-    for (int i = 0; i < 3; ++i) {
-      GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
-      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]);
-    }
-    drawRectangle(viewportX, viewportY, viewportWidth, viewportHeight);
-    // Unbind the textures as a precaution..
-    for (int i = 0; i < 3; ++i) {
-      GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i);
-      GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0);
-    }
-  }
-
-  private void drawRectangle(int x, int y, int width, int height) {
-    // Draw quad.
-    GLES20.glViewport(x, y, width, height);
-    GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4);
-  }
+  private static class ShaderCallbacks implements GlGenericDrawer.ShaderCallbacks {
+    @Override
+    public void onNewShader(GlShader shader) {}
 
-  private void prepareShader(String fragmentShader, float[] texMatrix) {
-    final Shader shader;
-    if (shaders.containsKey(fragmentShader)) {
-      shader = shaders.get(fragmentShader);
-    } else {
-      // Lazy allocation.
-      shader = new Shader(fragmentShader);
-      shaders.put(fragmentShader, shader);
-      shader.glShader.useProgram();
-      // Initialize fragment shader uniform values.
-      if (YUV_FRAGMENT_SHADER_STRING.equals(fragmentShader)) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("y_tex"), 0);
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("u_tex"), 1);
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("v_tex"), 2);
-      } else if (RGB_FRAGMENT_SHADER_STRING.equals(fragmentShader)) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("rgb_tex"), 0);
-      } else if (OES_FRAGMENT_SHADER_STRING.equals(fragmentShader)) {
-        GLES20.glUniform1i(shader.glShader.getUniformLocation("oes_tex"), 0);
-      } else {
-        throw new IllegalStateException("Unknown fragment shader: " + fragmentShader);
-      }
-      GlUtil.checkNoGLES2Error("Initialize fragment shader uniform values.");
-      // Initialize vertex shader attributes.
-      shader.glShader.setVertexAttribArray("in_pos", 2, FULL_RECTANGLE_BUF);
-      shader.glShader.setVertexAttribArray("in_tc", 2, FULL_RECTANGLE_TEX_BUF);
-    }
-    shader.glShader.useProgram();
-    // Copy the texture transformation matrix over.
-    GLES20.glUniformMatrix4fv(shader.texMatrixLocation, 1, false, texMatrix, 0);
+    @Override
+    public void onPrepareShader(GlShader shader, float[] texMatrix, int frameWidth, int frameHeight,
+        int viewportWidth, int viewportHeight) {}
   }
 
-  /**
-   * Release all GLES resources. This needs to be done manually, otherwise the resources are leaked.
-   */
-  @Override
-  public void release() {
-    for (Shader shader : shaders.values()) {
-      shader.glShader.release();
-    }
-    shaders.clear();
+  public GlRectDrawer() {
+    super(FRAGMENT_SHADER, new ShaderCallbacks());
   }
 }
diff --git a/sdk/android/api/org/webrtc/RendererCommon.java b/sdk/android/api/org/webrtc/RendererCommon.java
@@ -31,7 +31,12 @@ public static interface RendererEvents {
     public void onFrameResolutionChanged(int videoWidth, int videoHeight, int rotation);
   }
 
-  /** Interface for rendering frames on an EGLSurface. */
+  /**
+   * Interface for rendering frames on an EGLSurface with specified viewport location. Rotation,
+   * mirror, and cropping is specified using a 4x4 texture coordinate transform matrix. The frame
+   * input can either be an OES texture, RGB texture, or YUV textures in I420 format. The function
+   * release() must be called manually to free the resources held by this object.
+   */
   public static interface GlDrawer {
     /**
      * Functions for drawing frames with different sources. The rendering surface target is