From f7a726bb1146d9e954bfc9df700578144ecaff40 Mon Sep 17 00:00:00 2001 From: tonihei Date: Thu, 18 Jul 2024 08:27:39 -0700 Subject: [PATCH] Add MediaCodec loudness controller for API35+ This controller connects the audio output to the MediaCodec so that it can automatically propagate CTA-2075 loudness metadata. PiperOrigin-RevId: 653628503 --- RELEASENOTES.md | 2 + .../audio/MediaCodecAudioRenderer.java | 54 ++++++- .../AsynchronousMediaCodecAdapter.java | 20 ++- .../mediacodec/LoudnessCodecController.java | 137 ++++++++++++++++++ .../mediacodec/MediaCodecAdapter.java | 17 ++- .../SynchronousMediaCodecAdapter.java | 13 +- .../AsynchronousMediaCodecAdapterTest.java | 3 +- .../mediacodec/MediaCodecRendererTest.java | 2 +- 8 files changed, 236 insertions(+), 12 deletions(-) create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/LoudnessCodecController.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 86baab181c..8a4b3eeab9 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -38,6 +38,8 @@ extension 7 instead of API level 34 ([#1262](https://github.com/androidx/media/issues/1262)). * Audio: + * Automatically configure CTA-2075 loudness metadata on the codec if + present in the media. * Video: * `MediaCodecVideoRenderer` avoids decoding samples that are neither rendered nor used as reference by other samples. diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java index 84bdc6202e..915c78c156 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/audio/MediaCodecAudioRenderer.java @@ -58,6 +58,7 @@ import androidx.media3.exoplayer.audio.AudioRendererEventListener.EventDispatcher; import androidx.media3.exoplayer.audio.AudioSink.InitializationException; import androidx.media3.exoplayer.audio.AudioSink.WriteException; +import androidx.media3.exoplayer.mediacodec.LoudnessCodecController; import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter; import androidx.media3.exoplayer.mediacodec.MediaCodecInfo; import androidx.media3.exoplayer.mediacodec.MediaCodecRenderer; @@ -107,6 +108,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media private final Context context; private final EventDispatcher eventDispatcher; private final AudioSink audioSink; + @Nullable private final LoudnessCodecController loudnessCodecController; private int codecMaxInputSize; private boolean codecNeedsDiscardChannelsWorkaround; @@ -252,6 +254,43 @@ public MediaCodecAudioRenderer( @Nullable Handler eventHandler, @Nullable AudioRendererEventListener eventListener, AudioSink audioSink) { + this( + context, + codecAdapterFactory, + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + audioSink, + Util.SDK_INT >= 35 ? new LoudnessCodecController() : null); + } + + /** + * Creates a new instance. + * + * @param context A context. + * @param codecAdapterFactory The {@link MediaCodecAdapter.Factory} used to create {@link + * MediaCodecAdapter} instances. + * @param mediaCodecSelector A decoder selector. + * @param enableDecoderFallback Whether to enable fallback to lower-priority decoders if decoder + * initialization fails. This may result in using a decoder that is slower/less efficient than + * the primary decoder. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioSink The sink to which audio will be output. + * @param loudnessCodecController The {@link LoudnessCodecController}, or null to not control + * loudness. + */ + public MediaCodecAudioRenderer( + Context context, + MediaCodecAdapter.Factory codecAdapterFactory, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioSink audioSink, + @Nullable LoudnessCodecController loudnessCodecController) { super( C.TRACK_TYPE_AUDIO, codecAdapterFactory, @@ -261,6 +300,7 @@ public MediaCodecAudioRenderer( context = context.getApplicationContext(); this.context = context; this.audioSink = audioSink; + this.loudnessCodecController = loudnessCodecController; rendererPriority = C.PRIORITY_PLAYBACK; eventDispatcher = new EventDispatcher(eventHandler, eventListener); nextBufferToWritePresentationTimeUs = C.TIME_UNSET; @@ -443,7 +483,7 @@ protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( && !MimeTypes.AUDIO_RAW.equals(format.sampleMimeType); decryptOnlyCodecFormat = decryptOnlyCodecEnabled ? format : null; return MediaCodecAdapter.Configuration.createForAudioDecoding( - codecInfo, mediaFormat, format, crypto); + codecInfo, mediaFormat, format, crypto, loudnessCodecController); } @Override @@ -688,6 +728,9 @@ protected void onReset() { @Override protected void onRelease() { audioSink.release(); + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.release(); + } } @Override @@ -851,7 +894,7 @@ public void handleMessage(@MessageType int messageType, @Nullable Object message audioSink.setSkipSilenceEnabled((Boolean) checkNotNull(message)); break; case MSG_SET_AUDIO_SESSION_ID: - audioSink.setAudioSessionId((Integer) checkNotNull(message)); + setAudioSessionId((int) checkNotNull(message)); break; case MSG_SET_PRIORITY: rendererPriority = (int) checkNotNull(message); @@ -974,6 +1017,13 @@ protected MediaFormat getMediaFormat( return mediaFormat; } + private void setAudioSessionId(int audioSessionId) { + audioSink.setAudioSessionId(audioSessionId); + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.setAudioSessionId(audioSessionId); + } + } + private void updateCodecImportance() { @Nullable MediaCodecAdapter codec = getCodec(); if (codec == null) { diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java index 84272048ee..00b38ffc7d 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapter.java @@ -114,7 +114,11 @@ && useSynchronousBufferQueueingWithAsyncCryptoFlag(configuration.format)) { new AsynchronousMediaCodecBufferEnqueuer(codec, queueingThreadSupplier.get()); } codecAdapter = - new AsynchronousMediaCodecAdapter(codec, callbackThreadSupplier.get(), bufferEnqueuer); + new AsynchronousMediaCodecAdapter( + codec, + callbackThreadSupplier.get(), + bufferEnqueuer, + configuration.loudnessCodecController); TraceUtil.endSection(); if (configuration.surface == null && configuration.codecInfo.detachedSurfaceSupported @@ -157,14 +161,20 @@ private static boolean useSynchronousBufferQueueingWithAsyncCryptoFlag(Format fo private final MediaCodec codec; private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback; private final MediaCodecBufferEnqueuer bufferEnqueuer; + @Nullable private final LoudnessCodecController loudnessCodecController; + private boolean codecReleased; private @State int state; private AsynchronousMediaCodecAdapter( - MediaCodec codec, HandlerThread callbackThread, MediaCodecBufferEnqueuer bufferEnqueuer) { + MediaCodec codec, + HandlerThread callbackThread, + MediaCodecBufferEnqueuer bufferEnqueuer, + @Nullable LoudnessCodecController loudnessCodecController) { this.codec = codec; this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread); this.bufferEnqueuer = bufferEnqueuer; + this.loudnessCodecController = loudnessCodecController; this.state = STATE_CREATED; } @@ -181,6 +191,9 @@ private void initialize( TraceUtil.beginSection("startCodec"); codec.start(); TraceUtil.endSection(); + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.addMediaCodec(codec); + } state = STATE_INITIALIZED; } @@ -273,6 +286,9 @@ public void release() { codec.stop(); } } finally { + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.removeMediaCodec(codec); + } codec.release(); codecReleased = true; } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/LoudnessCodecController.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/LoudnessCodecController.java new file mode 100644 index 0000000000..e6ca71ec04 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/LoudnessCodecController.java @@ -0,0 +1,137 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.mediacodec; + +import static androidx.media3.common.util.Assertions.checkState; +import static com.google.common.util.concurrent.MoreExecutors.directExecutor; + +import android.media.LoudnessCodecController.OnLoudnessCodecUpdateListener; +import android.media.MediaCodec; +import android.os.Bundle; +import androidx.annotation.DoNotInline; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.media3.common.util.UnstableApi; +import java.util.HashSet; +import java.util.Iterator; + +/** Wrapper class for the platform {@link android.media.LoudnessCodecController}. */ +@RequiresApi(35) +@UnstableApi +public final class LoudnessCodecController { + + /** Interface to intercept and modify loudness parameters before applying them to the codec. */ + public interface LoudnessParameterUpdateListener { + + /** The default update listener returning an unmodified set of parameters. */ + LoudnessParameterUpdateListener DEFAULT = bundle -> bundle; + + /** + * Returns the updated loudness parameters to be applied to the codec. + * + * @param parameters The suggested loudness parameters. + * @return The updated loudness parameters. + */ + Bundle onLoudnessParameterUpdate(Bundle parameters); + } + + private final HashSet mediaCodecs; + private final LoudnessParameterUpdateListener updateListener; + + @Nullable private android.media.LoudnessCodecController loudnessCodecController; + + /** Creates the loudness controller. */ + public LoudnessCodecController() { + this(LoudnessParameterUpdateListener.DEFAULT); + } + + /** + * Creates the loudness controller. + * + * @param updateListener The {@link LoudnessParameterUpdateListener} to intercept and modify + * parameters. + */ + public LoudnessCodecController(LoudnessParameterUpdateListener updateListener) { + this.mediaCodecs = new HashSet<>(); + this.updateListener = updateListener; + } + + /** + * Configures the loudness controller with an audio session id. + * + * @param audioSessionId The audio session ID. + */ + @DoNotInline + public void setAudioSessionId(int audioSessionId) { + if (loudnessCodecController != null) { + loudnessCodecController.close(); + loudnessCodecController = null; + } + android.media.LoudnessCodecController loudnessCodecController = + android.media.LoudnessCodecController.create( + audioSessionId, + directExecutor(), + new OnLoudnessCodecUpdateListener() { + @Override + public Bundle onLoudnessCodecUpdate(MediaCodec codec, Bundle parameters) { + return updateListener.onLoudnessParameterUpdate(parameters); + } + }); + this.loudnessCodecController = loudnessCodecController; + for (Iterator it = mediaCodecs.iterator(); it.hasNext(); ) { + boolean registered = loudnessCodecController.addMediaCodec(it.next()); + if (!registered) { + it.remove(); + } + } + } + + /** + * Adds a codec to be configured by the loudness controller. + * + * @param mediaCodec A {@link MediaCodec}. + */ + @DoNotInline + public void addMediaCodec(MediaCodec mediaCodec) { + if (loudnessCodecController != null && !loudnessCodecController.addMediaCodec(mediaCodec)) { + // Don't add codec if the existing loudness controller can't handle it. + return; + } + checkState(mediaCodecs.add(mediaCodec)); + } + + /** + * Removes a codec from being configured by the loudness controller. + * + * @param mediaCodec A {@link MediaCodec}. + */ + @DoNotInline + public void removeMediaCodec(MediaCodec mediaCodec) { + boolean removedCodec = mediaCodecs.remove(mediaCodec); + if (removedCodec && loudnessCodecController != null) { + loudnessCodecController.removeMediaCodec(mediaCodec); + } + } + + /** Releases the loudness controller. */ + @DoNotInline + public void release() { + mediaCodecs.clear(); + if (loudnessCodecController != null) { + loudnessCodecController.close(); + } + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java index 509f6905f3..c8c628c674 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/MediaCodecAdapter.java @@ -50,14 +50,17 @@ final class Configuration { * @param mediaFormat See {@link #mediaFormat}. * @param format See {@link #format}. * @param crypto See {@link #crypto}. + * @param loudnessCodecController See {@link #loudnessCodecController}. * @return The created instance. */ public static Configuration createForAudioDecoding( MediaCodecInfo codecInfo, MediaFormat mediaFormat, Format format, - @Nullable MediaCrypto crypto) { - return new Configuration(codecInfo, mediaFormat, format, /* surface= */ null, crypto); + @Nullable MediaCrypto crypto, + @Nullable LoudnessCodecController loudnessCodecController) { + return new Configuration( + codecInfo, mediaFormat, format, /* surface= */ null, crypto, loudnessCodecController); } /** @@ -76,7 +79,8 @@ public static Configuration createForVideoDecoding( Format format, @Nullable Surface surface, @Nullable MediaCrypto crypto) { - return new Configuration(codecInfo, mediaFormat, format, surface, crypto); + return new Configuration( + codecInfo, mediaFormat, format, surface, crypto, /* loudnessCodecController= */ null); } /** Information about the {@link MediaCodec} being configured. */ @@ -98,17 +102,22 @@ public static Configuration createForVideoDecoding( /** For DRM protected playbacks, a {@link MediaCrypto} to use for decryption. */ @Nullable public final MediaCrypto crypto; + /** The {@link LoudnessCodecController} for audio codecs. */ + @Nullable public final LoudnessCodecController loudnessCodecController; + private Configuration( MediaCodecInfo codecInfo, MediaFormat mediaFormat, Format format, @Nullable Surface surface, - @Nullable MediaCrypto crypto) { + @Nullable MediaCrypto crypto, + @Nullable LoudnessCodecController loudnessCodecController) { this.codecInfo = codecInfo; this.mediaFormat = mediaFormat; this.format = format; this.surface = surface; this.crypto = crypto; + this.loudnessCodecController = loudnessCodecController; } } diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java index 88c80f12d6..8e9b91d7a6 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/mediacodec/SynchronousMediaCodecAdapter.java @@ -63,7 +63,7 @@ public MediaCodecAdapter createAdapter(Configuration configuration) throws IOExc TraceUtil.beginSection("startCodec"); codec.start(); TraceUtil.endSection(); - return new SynchronousMediaCodecAdapter(codec); + return new SynchronousMediaCodecAdapter(codec, configuration.loudnessCodecController); } catch (IOException | RuntimeException e) { if (codec != null) { codec.release(); @@ -84,9 +84,15 @@ protected MediaCodec createCodec(Configuration configuration) throws IOException } private final MediaCodec codec; + @Nullable private final LoudnessCodecController loudnessCodecController; - private SynchronousMediaCodecAdapter(MediaCodec mediaCodec) { + private SynchronousMediaCodecAdapter( + MediaCodec mediaCodec, @Nullable LoudnessCodecController loudnessCodecController) { this.codec = mediaCodec; + this.loudnessCodecController = loudnessCodecController; + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.addMediaCodec(codec); + } } @Override @@ -165,6 +171,9 @@ public void release() { codec.stop(); } } finally { + if (Util.SDK_INT >= 35 && loudnessCodecController != null) { + loudnessCodecController.removeMediaCodec(codec); + } codec.release(); } } diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapterTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapterTest.java index 1656f0b0fe..6c8705d496 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapterTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/AsynchronousMediaCodecAdapterTest.java @@ -47,7 +47,8 @@ public void setUp() throws Exception { codecInfo, createMediaFormat("format"), new Format.Builder().build(), - /* crypto= */ null); + /* crypto= */ null, + /* loudnessCodecController= */ null); callbackThread = new HandlerThread("TestCallbackThread"); queueingThread = new HandlerThread("TestQueueingThread"); adapter = diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java index cec2df3a1d..e19f5e7edb 100644 --- a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/mediacodec/MediaCodecRendererTest.java @@ -599,7 +599,7 @@ protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( @Nullable MediaCrypto crypto, float codecOperatingRate) { return MediaCodecAdapter.Configuration.createForAudioDecoding( - codecInfo, new MediaFormat(), format, crypto); + codecInfo, new MediaFormat(), format, crypto, /* loudnessCodecController= */ null); } @Override