HDR: Input ColorInfo to the FrameProcessor.

This allows the GlEffectsFrameProcessor to later handle HLG and PQ differently, or limited and full color range differently. No functional change intended in this CL. PiperOrigin-RevId: 466070764
DiceTechnology · Oct 19, 2022 · e444eaa · e444eaa
1 parent 3aa99d5
commit e444eaa
Show file tree

Hide file tree

Showing 7 changed files with 157 additions and 81 deletions.
diff --git a/.../effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java b/.../effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
@@ -42,6 +42,7 @@
 import androidx.media3.common.SurfaceInfo;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
 import java.util.List;
@@ -388,7 +389,7 @@ public void onFrameProcessingEnded() {
                       },
                       effects,
                       DebugViewProvider.NONE,
-                      /* useHdr= */ false));
+                      ColorInfo.SDR_BT709_LIMITED));
       glEffectsFrameProcessor.setInputFrameInfo(
           new FrameInfo(inputWidth, inputHeight, pixelWidthHeightRatio, /* streamOffsetUs= */ 0));
       glEffectsFrameProcessor.registerInputFrame();

diff --git a/...ffect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java b/...ffect/src/main/java/androidx/media3/effect/FinalMatrixTransformationProcessorWrapper.java
@@ -40,6 +40,7 @@
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Log;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import java.util.ArrayDeque;
 import java.util.Queue;
@@ -69,7 +70,7 @@
   private final DebugViewProvider debugViewProvider;
   private final FrameProcessor.Listener frameProcessorListener;
   private final boolean sampleFromExternalTexture;
-  private final boolean useHdr;
+  private final ColorInfo colorInfo;
   private final float[] textureTransformMatrix;
   private final Queue<Long> streamOffsetUsQueue;
 
@@ -91,8 +92,6 @@
   @Nullable
   private EGLSurface outputEglSurface;
 
-  // TODO(b/227624622): Instead of inputting useHdr, input ColorInfo to handle HLG and PQ
-  //  differently.
   public FinalMatrixTransformationProcessorWrapper(
       Context context,
       EGLDisplay eglDisplay,
@@ -101,15 +100,15 @@ public FinalMatrixTransformationProcessorWrapper(
       FrameProcessor.Listener frameProcessorListener,
       DebugViewProvider debugViewProvider,
       boolean sampleFromExternalTexture,
-      boolean useHdr) {
+      ColorInfo colorInfo) {
     this.context = context;
     this.matrixTransformations = matrixTransformations;
     this.eglDisplay = eglDisplay;
     this.eglContext = eglContext;
     this.debugViewProvider = debugViewProvider;
     this.frameProcessorListener = frameProcessorListener;
     this.sampleFromExternalTexture = sampleFromExternalTexture;
-    this.useHdr = useHdr;
+    this.colorInfo = colorInfo;
 
     textureTransformMatrix = new float[16];
     Matrix.setIdentityM(textureTransformMatrix, /* smOffset= */ 0);
@@ -215,7 +214,8 @@ private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
     SurfaceInfo outputSurfaceInfo = this.outputSurfaceInfo;
     @Nullable EGLSurface outputEglSurface = this.outputEglSurface;
     if (outputEglSurface == null) {
-      if (useHdr) {
+      boolean colorInfoIsHdr = ColorInfo.isHdr(colorInfo);
+      if (colorInfoIsHdr) {
         outputEglSurface = GlUtil.getEglSurfaceRgba1010102(eglDisplay, outputSurfaceInfo.surface);
       } else {
         outputEglSurface = GlUtil.getEglSurface(eglDisplay, outputSurfaceInfo.surface);
@@ -227,7 +227,7 @@ private synchronized boolean ensureConfigured(int inputWidth, int inputHeight)
               outputSurfaceInfo.width, outputSurfaceInfo.height);
       if (debugSurfaceView != null && !Util.areEqual(this.debugSurfaceView, debugSurfaceView)) {
         debugSurfaceViewWrapper =
-            new SurfaceViewWrapper(eglDisplay, eglContext, useHdr, debugSurfaceView);
+            new SurfaceViewWrapper(eglDisplay, eglContext, colorInfoIsHdr, debugSurfaceView);
       }
       this.debugSurfaceView = debugSurfaceView;
     }
@@ -266,7 +266,7 @@ private MatrixTransformationProcessor createMatrixTransformationProcessorForOutp
             context,
             matrixTransformationListBuilder.build(),
             sampleFromExternalTexture,
-            useHdr,
+            colorInfo,
             /* outputOpticalColors= */ true);
     matrixTransformationProcessor.setTextureTransformMatrix(textureTransformMatrix);
     Pair<Integer, Integer> outputSize =

diff --git a/.../media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java b/.../media/libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
@@ -37,6 +37,7 @@
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.util.GlUtil;
 import com.google.android.exoplayer2.util.Util;
+import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -66,7 +67,7 @@ public GlEffectsFrameProcessor create(
         FrameProcessor.Listener listener,
         List<Effect> effects,
         DebugViewProvider debugViewProvider,
-        boolean useHdr)
+        ColorInfo colorInfo)
         throws FrameProcessingException {
 
       ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
@@ -79,7 +80,7 @@ public GlEffectsFrameProcessor create(
                       listener,
                       effects,
                       debugViewProvider,
-                      useHdr,
+                      colorInfo,
                       singleThreadExecutorService));
 
       try {
@@ -109,11 +110,14 @@ private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor(
       FrameProcessor.Listener listener,
       List<Effect> effects,
       DebugViewProvider debugViewProvider,
-      boolean useHdr,
+      ColorInfo colorInfo,
       ExecutorService singleThreadExecutorService)
       throws GlUtil.GlException, FrameProcessingException {
     checkState(Thread.currentThread().getName().equals(THREAD_NAME));
 
+    // TODO(b/237674316): Delay initialization of things requiring the colorInfo, to
+    //  configure based on the color info from the decoder output media format instead.
+    boolean useHdr = ColorInfo.isHdr(colorInfo);
     EGLDisplay eglDisplay = GlUtil.createEglDisplay();
     EGLContext eglContext =
         useHdr
@@ -131,7 +135,7 @@ private static GlEffectsFrameProcessor createOpenGlObjectsAndFrameProcessor(
 
     ImmutableList<GlTextureProcessor> textureProcessors =
         getGlTextureProcessorsForGlEffects(
-            context, effects, eglDisplay, eglContext, listener, debugViewProvider, useHdr);
+            context, effects, eglDisplay, eglContext, listener, debugViewProvider, colorInfo);
     FrameProcessingTaskExecutor frameProcessingTaskExecutor =
         new FrameProcessingTaskExecutor(singleThreadExecutorService, listener);
     chainTextureProcessorsWithListeners(textureProcessors, frameProcessingTaskExecutor, listener);
@@ -162,7 +166,7 @@ private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffe
       EGLContext eglContext,
       FrameProcessor.Listener listener,
       DebugViewProvider debugViewProvider,
-      boolean useHdr)
+      ColorInfo colorInfo)
       throws FrameProcessingException {
     ImmutableList.Builder<GlTextureProcessor> textureProcessorListBuilder =
         new ImmutableList.Builder<>();
@@ -185,12 +189,13 @@ private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffe
                 context,
                 matrixTransformations,
                 sampleFromExternalTexture,
-                useHdr,
+                colorInfo,
                 /* outputOpticalColors= */ false));
         matrixTransformationListBuilder = new ImmutableList.Builder<>();
         sampleFromExternalTexture = false;
       }
-      textureProcessorListBuilder.add(glEffect.toGlTextureProcessor(context, useHdr));
+      textureProcessorListBuilder.add(
+          glEffect.toGlTextureProcessor(context, ColorInfo.isHdr(colorInfo)));
     }
     textureProcessorListBuilder.add(
         new FinalMatrixTransformationProcessorWrapper(
@@ -201,7 +206,7 @@ private static ImmutableList<GlTextureProcessor> getGlTextureProcessorsForGlEffe
             listener,
             debugViewProvider,
             sampleFromExternalTexture,
-            useHdr));
+            colorInfo));
     return textureProcessorListBuilder.build();
   }
 

diff --git a/.../libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java b/.../libraries/effect/src/main/java/androidx/media3/effect/MatrixTransformationProcessor.java
@@ -24,6 +24,7 @@
 import androidx.media3.common.FrameProcessingException;
 import com.google.android.exoplayer2.util.GlProgram;
 import com.google.android.exoplayer2.util.GlUtil;
+import com.google.android.exoplayer2.video.ColorInfo;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.util.Arrays;
@@ -110,11 +111,13 @@ public MatrixTransformationProcessor(
       Context context, boolean useHdr, MatrixTransformation matrixTransformation)
       throws FrameProcessingException {
     this(
-        context,
+        createGlProgram(
+            context,
+            /* inputOpticalColorsFromExternalTexture= */ false,
+            useHdr,
+            /* outputOpticalColors= */ false),
         ImmutableList.of(matrixTransformation),
-        /* sampleFromExternalTexture= */ false,
-        useHdr,
-        /* outputOpticalColors= */ false);
+        useHdr);
   }
 
   /**
@@ -131,86 +134,134 @@ public MatrixTransformationProcessor(
       Context context, boolean useHdr, GlMatrixTransformation matrixTransformation)
       throws FrameProcessingException {
     this(
-        context,
+        createGlProgram(
+            context,
+            /* inputOpticalColorsFromExternalTexture= */ false,
+            useHdr,
+            /* outputOpticalColors= */ false),
         ImmutableList.of(matrixTransformation),
-        /* sampleFromExternalTexture= */ false,
-        useHdr,
-        /* outputOpticalColors= */ false);
+        useHdr);
   }
 
   /**
    * Creates a new instance.
    *
+   * <p>Able to convert optical {@link ColorInfo} inputs and outputs to and from the intermediate
+   * {@link GlTextureProcessor} colors of linear RGB BT.2020 for HDR, and gamma RGB BT.709 for SDR.
+   *
    * @param context The {@link Context}.
    * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
    *     apply to each frame in order.
-   * @param sampleFromExternalTexture Whether the input will be provided using an external texture.
-   *     If {@code true}, the caller should use {@link #setTextureTransformMatrix(float[])} to
-   *     provide the transformation matrix associated with the external texture.
-   * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
-   *     EXT_YUV_target} OpenGL extension.
-   * @param outputOpticalColors If {@code true} and {@code useHdr} is also {@code true}, outputs a
-   *     non-linear optical, or display light colors, possibly by applying the EOTF (Electro-optical
-   *     transfer function). Otherwise, outputs linear electrical colors.
+   * @param inputOpticalColorsFromExternalTexture Whether optical color input will be provided using
+   *     an external texture. If {@code true}, the caller should use {@link
+   *     #setTextureTransformMatrix(float[])} to provide the transformation matrix associated with
+   *     the external texture.
+   * @param opticalColorInfo The optical {@link ColorInfo}, only used to transform between color
+   *     spaces and transfers, when {@code inputOpticalColorsFromExternalTexture} or {@code
+   *     outputOpticalColors} are {@code true}. If it {@link ColorInfo#isHdr(ColorInfo)},
+   *     intermediate {@link GlTextureProcessor} colors will be in linear RGB BT.2020. Otherwise,
+   *     these colors will be in gamma RGB BT.709.
+   * @param outputOpticalColors If {@code true}, outputs {@code opticalColorInfo}. If {@code false},
+   *     outputs intermediate colors of linear RGB BT.2020 if {@code opticalColorInfo} {@link
+   *     ColorInfo#isHdr(ColorInfo)}, and gamma RGB BT.709 otherwise.
    * @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
    *     operation fails or is unsupported.
    */
   public MatrixTransformationProcessor(
       Context context,
       ImmutableList<GlMatrixTransformation> matrixTransformations,
-      boolean sampleFromExternalTexture,
-      boolean useHdr,
+      boolean inputOpticalColorsFromExternalTexture,
+      ColorInfo opticalColorInfo,
       boolean outputOpticalColors)
       throws FrameProcessingException {
-    super(useHdr);
-    if (sampleFromExternalTexture && useHdr && !GlUtil.isYuvTargetExtensionSupported()) {
+    this(
+        createGlProgram(
+            context,
+            inputOpticalColorsFromExternalTexture,
+            ColorInfo.isHdr(opticalColorInfo),
+            outputOpticalColors),
+        matrixTransformations,
+        ColorInfo.isHdr(opticalColorInfo));
+    if (!ColorInfo.isHdr(opticalColorInfo) || !inputOpticalColorsFromExternalTexture) {
+      return;
+    }
+    // TODO(b/227624622): Implement YUV to RGB conversions in COLOR_RANGE_LIMITED as well, using
+    //  opticalColorInfo.colorRange to select between them.
+
+    // In HDR editing mode the decoder output is sampled in YUV.
+    if (!GlUtil.isYuvTargetExtensionSupported()) {
       throw new FrameProcessingException(
           "The EXT_YUV_target extension is required for HDR editing input.");
     }
+    glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
+    // TODO(b/227624622): Implement PQ and gamma TFs, and use an @IntDef to select between HLG,
+    //  PQ, and gamma, coming from opticalColorInfo.colorTransfer.
+
+    // Applying the OETF will output a linear signal. Not applying the OETF will output an optical
+    // signal.
+    glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
+  }
 
+  /**
+   * Creates a new instance.
+   *
+   * @param glProgram The {@link GlProgram}.
+   * @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
+   *     apply to each frame in order.
+   * @param useHdr Whether to process the input as an HDR signal. Using HDR requires the {@code
+   *     EXT_YUV_target} OpenGL extension.
+   */
+  private MatrixTransformationProcessor(
+      GlProgram glProgram,
+      ImmutableList<GlMatrixTransformation> matrixTransformations,
+      boolean useHdr) {
+    super(useHdr);
+    this.glProgram = glProgram;
     this.matrixTransformations = matrixTransformations;
 
     transformationMatrixCache = new float[matrixTransformations.size()][16];
     compositeTransformationMatrix = new float[16];
     tempResultMatrix = new float[16];
     Matrix.setIdentityM(compositeTransformationMatrix, /* smOffset= */ 0);
     visiblePolygon = NDC_SQUARE;
+  }
+
+  private static GlProgram createGlProgram(
+      Context context,
+      boolean inputOpticalColorsFromExternalTexture,
+      boolean useHdr,
+      boolean outputOpticalColors)
+      throws FrameProcessingException {
 
     String vertexShaderFilePath;
     String fragmentShaderFilePath;
-    if (sampleFromExternalTexture) {
-      vertexShaderFilePath =
-          useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
-      fragmentShaderFilePath =
-          useHdr ? FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH : FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
-    } else if (outputOpticalColors) {
-      vertexShaderFilePath =
-          useHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
-      fragmentShaderFilePath =
-          useHdr ? FRAGMENT_SHADER_HLG_EOTF_ES3_PATH : FRAGMENT_SHADER_COPY_PATH;
+    if (inputOpticalColorsFromExternalTexture) {
+      if (useHdr) {
+        vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
+        fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_YUV_ES3_PATH;
+      } else {
+        vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
+        fragmentShaderFilePath = FRAGMENT_SHADER_COPY_EXTERNAL_PATH;
+      }
+    } else if (outputOpticalColors && useHdr) {
+      vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_ES3_PATH;
+      fragmentShaderFilePath = FRAGMENT_SHADER_HLG_EOTF_ES3_PATH;
     } else {
       vertexShaderFilePath = VERTEX_SHADER_TRANSFORMATION_PATH;
       fragmentShaderFilePath = FRAGMENT_SHADER_COPY_PATH;
     }
 
+    GlProgram glProgram;
     try {
       glProgram = new GlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
     } catch (IOException | GlUtil.GlException e) {
       throw new FrameProcessingException(e);
     }
 
-    if (useHdr && sampleFromExternalTexture) {
-      // In HDR editing mode the decoder output is sampled in YUV.
-      glProgram.setFloatsUniform("uColorTransform", MATRIX_YUV_TO_BT2020_COLOR_TRANSFORM);
-      // TODO(b/227624622): Implement PQ, and use an @IntDef to select between HLG, PQ, and no
-      //  transfer function.
-      // Applying the OETF will output a linear signal. Not applying the OETF will output an optical
-      // signal.
-      glProgram.setFloatUniform("uApplyHlgOetf", outputOpticalColors ? 0.0f : 1.0f);
-    }
     float[] identityMatrix = new float[16];
     Matrix.setIdentityM(identityMatrix, /* smOffset= */ 0);
     glProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
+    return glProgram;
   }
 
   @Override
@@ -274,11 +325,11 @@ private void updateCompositeTransformationMatrixAndVisiblePolygon(long presentat
     visiblePolygon = NDC_SQUARE;
     for (float[] transformationMatrix : transformationMatrixCache) {
       Matrix.multiplyMM(
-          tempResultMatrix,
+          /* result= */ tempResultMatrix,
           /* resultOffset= */ 0,
-          transformationMatrix,
+          /* lhs= */ transformationMatrix,
           /* lhsOffset= */ 0,
-          compositeTransformationMatrix,
+          /* rhs= */ compositeTransformationMatrix,
           /* rhsOffset= */ 0);
       System.arraycopy(
           /* src= */ tempResultMatrix,