Skip to content

Commit

Permalink
Add MediaCodec loudness controller for API35+
Browse files Browse the repository at this point in the history
This controller connects the audio output to the MediaCodec so
that it can automatically propagate CTA-2075 loudness metadata.

PiperOrigin-RevId: 653628503
  • Loading branch information
tonihei authored and copybara-github committed Jul 18, 2024
1 parent a52df6d commit f7a726b
Show file tree
Hide file tree
Showing 8 changed files with 236 additions and 12 deletions.
2 changes: 2 additions & 0 deletions RELEASENOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
extension 7 instead of API level 34
([#1262](https://github.com/androidx/media/issues/1262)).
* Audio:
* Automatically configure CTA-2075 loudness metadata on the codec if
present in the media.
* Video:
* `MediaCodecVideoRenderer` avoids decoding samples that are neither
rendered nor used as reference by other samples.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import androidx.media3.exoplayer.audio.AudioRendererEventListener.EventDispatcher;
import androidx.media3.exoplayer.audio.AudioSink.InitializationException;
import androidx.media3.exoplayer.audio.AudioSink.WriteException;
import androidx.media3.exoplayer.mediacodec.LoudnessCodecController;
import androidx.media3.exoplayer.mediacodec.MediaCodecAdapter;
import androidx.media3.exoplayer.mediacodec.MediaCodecInfo;
import androidx.media3.exoplayer.mediacodec.MediaCodecRenderer;
Expand Down Expand Up @@ -107,6 +108,7 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media
private final Context context;
private final EventDispatcher eventDispatcher;
private final AudioSink audioSink;
@Nullable private final LoudnessCodecController loudnessCodecController;

private int codecMaxInputSize;
private boolean codecNeedsDiscardChannelsWorkaround;
Expand Down Expand Up @@ -252,6 +254,43 @@ public MediaCodecAudioRenderer(
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioSink audioSink) {
this(
context,
codecAdapterFactory,
mediaCodecSelector,
enableDecoderFallback,
eventHandler,
eventListener,
audioSink,
Util.SDK_INT >= 35 ? new LoudnessCodecController() : null);
}

/**
* Creates a new instance.
*
* @param context A context.
* @param codecAdapterFactory The {@link MediaCodecAdapter.Factory} used to create {@link
* MediaCodecAdapter} instances.
* @param mediaCodecSelector A decoder selector.
* @param enableDecoderFallback Whether to enable fallback to lower-priority decoders if decoder
* initialization fails. This may result in using a decoder that is slower/less efficient than
* the primary decoder.
* @param eventHandler A handler to use when delivering events to {@code eventListener}. May be
* null if delivery of events is not required.
* @param eventListener A listener of events. May be null if delivery of events is not required.
* @param audioSink The sink to which audio will be output.
* @param loudnessCodecController The {@link LoudnessCodecController}, or null to not control
* loudness.
*/
public MediaCodecAudioRenderer(
Context context,
MediaCodecAdapter.Factory codecAdapterFactory,
MediaCodecSelector mediaCodecSelector,
boolean enableDecoderFallback,
@Nullable Handler eventHandler,
@Nullable AudioRendererEventListener eventListener,
AudioSink audioSink,
@Nullable LoudnessCodecController loudnessCodecController) {
super(
C.TRACK_TYPE_AUDIO,
codecAdapterFactory,
Expand All @@ -261,6 +300,7 @@ public MediaCodecAudioRenderer(
context = context.getApplicationContext();
this.context = context;
this.audioSink = audioSink;
this.loudnessCodecController = loudnessCodecController;
rendererPriority = C.PRIORITY_PLAYBACK;
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
nextBufferToWritePresentationTimeUs = C.TIME_UNSET;
Expand Down Expand Up @@ -443,7 +483,7 @@ protected MediaCodecAdapter.Configuration getMediaCodecConfiguration(
&& !MimeTypes.AUDIO_RAW.equals(format.sampleMimeType);
decryptOnlyCodecFormat = decryptOnlyCodecEnabled ? format : null;
return MediaCodecAdapter.Configuration.createForAudioDecoding(
codecInfo, mediaFormat, format, crypto);
codecInfo, mediaFormat, format, crypto, loudnessCodecController);
}

@Override
Expand Down Expand Up @@ -688,6 +728,9 @@ protected void onReset() {
@Override
protected void onRelease() {
audioSink.release();
if (Util.SDK_INT >= 35 && loudnessCodecController != null) {
loudnessCodecController.release();
}
}

@Override
Expand Down Expand Up @@ -851,7 +894,7 @@ public void handleMessage(@MessageType int messageType, @Nullable Object message
audioSink.setSkipSilenceEnabled((Boolean) checkNotNull(message));
break;
case MSG_SET_AUDIO_SESSION_ID:
audioSink.setAudioSessionId((Integer) checkNotNull(message));
setAudioSessionId((int) checkNotNull(message));
break;
case MSG_SET_PRIORITY:
rendererPriority = (int) checkNotNull(message);
Expand Down Expand Up @@ -974,6 +1017,13 @@ protected MediaFormat getMediaFormat(
return mediaFormat;
}

private void setAudioSessionId(int audioSessionId) {
audioSink.setAudioSessionId(audioSessionId);
if (Util.SDK_INT >= 35 && loudnessCodecController != null) {
loudnessCodecController.setAudioSessionId(audioSessionId);
}
}

private void updateCodecImportance() {
@Nullable MediaCodecAdapter codec = getCodec();
if (codec == null) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,11 @@ && useSynchronousBufferQueueingWithAsyncCryptoFlag(configuration.format)) {
new AsynchronousMediaCodecBufferEnqueuer(codec, queueingThreadSupplier.get());
}
codecAdapter =
new AsynchronousMediaCodecAdapter(codec, callbackThreadSupplier.get(), bufferEnqueuer);
new AsynchronousMediaCodecAdapter(
codec,
callbackThreadSupplier.get(),
bufferEnqueuer,
configuration.loudnessCodecController);
TraceUtil.endSection();
if (configuration.surface == null
&& configuration.codecInfo.detachedSurfaceSupported
Expand Down Expand Up @@ -157,14 +161,20 @@ private static boolean useSynchronousBufferQueueingWithAsyncCryptoFlag(Format fo
private final MediaCodec codec;
private final AsynchronousMediaCodecCallback asynchronousMediaCodecCallback;
private final MediaCodecBufferEnqueuer bufferEnqueuer;
@Nullable private final LoudnessCodecController loudnessCodecController;

private boolean codecReleased;
private @State int state;

private AsynchronousMediaCodecAdapter(
MediaCodec codec, HandlerThread callbackThread, MediaCodecBufferEnqueuer bufferEnqueuer) {
MediaCodec codec,
HandlerThread callbackThread,
MediaCodecBufferEnqueuer bufferEnqueuer,
@Nullable LoudnessCodecController loudnessCodecController) {
this.codec = codec;
this.asynchronousMediaCodecCallback = new AsynchronousMediaCodecCallback(callbackThread);
this.bufferEnqueuer = bufferEnqueuer;
this.loudnessCodecController = loudnessCodecController;
this.state = STATE_CREATED;
}

Expand All @@ -181,6 +191,9 @@ private void initialize(
TraceUtil.beginSection("startCodec");
codec.start();
TraceUtil.endSection();
if (Util.SDK_INT >= 35 && loudnessCodecController != null) {
loudnessCodecController.addMediaCodec(codec);
}
state = STATE_INITIALIZED;
}

Expand Down Expand Up @@ -273,6 +286,9 @@ public void release() {
codec.stop();
}
} finally {
if (Util.SDK_INT >= 35 && loudnessCodecController != null) {
loudnessCodecController.removeMediaCodec(codec);
}
codec.release();
codecReleased = true;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.mediacodec;

import static androidx.media3.common.util.Assertions.checkState;
import static com.google.common.util.concurrent.MoreExecutors.directExecutor;

import android.media.LoudnessCodecController.OnLoudnessCodecUpdateListener;
import android.media.MediaCodec;
import android.os.Bundle;
import androidx.annotation.DoNotInline;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import androidx.media3.common.util.UnstableApi;
import java.util.HashSet;
import java.util.Iterator;

/** Wrapper class for the platform {@link android.media.LoudnessCodecController}. */
@RequiresApi(35)
@UnstableApi
public final class LoudnessCodecController {

/** Interface to intercept and modify loudness parameters before applying them to the codec. */
public interface LoudnessParameterUpdateListener {

/** The default update listener returning an unmodified set of parameters. */
LoudnessParameterUpdateListener DEFAULT = bundle -> bundle;

/**
* Returns the updated loudness parameters to be applied to the codec.
*
* @param parameters The suggested loudness parameters.
* @return The updated loudness parameters.
*/
Bundle onLoudnessParameterUpdate(Bundle parameters);
}

private final HashSet<MediaCodec> mediaCodecs;
private final LoudnessParameterUpdateListener updateListener;

@Nullable private android.media.LoudnessCodecController loudnessCodecController;

/** Creates the loudness controller. */
public LoudnessCodecController() {
this(LoudnessParameterUpdateListener.DEFAULT);
}

/**
* Creates the loudness controller.
*
* @param updateListener The {@link LoudnessParameterUpdateListener} to intercept and modify
* parameters.
*/
public LoudnessCodecController(LoudnessParameterUpdateListener updateListener) {
this.mediaCodecs = new HashSet<>();
this.updateListener = updateListener;
}

/**
* Configures the loudness controller with an audio session id.
*
* @param audioSessionId The audio session ID.
*/
@DoNotInline
public void setAudioSessionId(int audioSessionId) {
if (loudnessCodecController != null) {
loudnessCodecController.close();
loudnessCodecController = null;
}
android.media.LoudnessCodecController loudnessCodecController =
android.media.LoudnessCodecController.create(
audioSessionId,
directExecutor(),
new OnLoudnessCodecUpdateListener() {
@Override
public Bundle onLoudnessCodecUpdate(MediaCodec codec, Bundle parameters) {
return updateListener.onLoudnessParameterUpdate(parameters);
}
});
this.loudnessCodecController = loudnessCodecController;
for (Iterator<MediaCodec> it = mediaCodecs.iterator(); it.hasNext(); ) {
boolean registered = loudnessCodecController.addMediaCodec(it.next());
if (!registered) {
it.remove();
}
}
}

/**
* Adds a codec to be configured by the loudness controller.
*
* @param mediaCodec A {@link MediaCodec}.
*/
@DoNotInline
public void addMediaCodec(MediaCodec mediaCodec) {
if (loudnessCodecController != null && !loudnessCodecController.addMediaCodec(mediaCodec)) {
// Don't add codec if the existing loudness controller can't handle it.
return;
}
checkState(mediaCodecs.add(mediaCodec));
}

/**
* Removes a codec from being configured by the loudness controller.
*
* @param mediaCodec A {@link MediaCodec}.
*/
@DoNotInline
public void removeMediaCodec(MediaCodec mediaCodec) {
boolean removedCodec = mediaCodecs.remove(mediaCodec);
if (removedCodec && loudnessCodecController != null) {
loudnessCodecController.removeMediaCodec(mediaCodec);
}
}

/** Releases the loudness controller. */
@DoNotInline
public void release() {
mediaCodecs.clear();
if (loudnessCodecController != null) {
loudnessCodecController.close();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,17 @@ final class Configuration {
* @param mediaFormat See {@link #mediaFormat}.
* @param format See {@link #format}.
* @param crypto See {@link #crypto}.
* @param loudnessCodecController See {@link #loudnessCodecController}.
* @return The created instance.
*/
public static Configuration createForAudioDecoding(
MediaCodecInfo codecInfo,
MediaFormat mediaFormat,
Format format,
@Nullable MediaCrypto crypto) {
return new Configuration(codecInfo, mediaFormat, format, /* surface= */ null, crypto);
@Nullable MediaCrypto crypto,
@Nullable LoudnessCodecController loudnessCodecController) {
return new Configuration(
codecInfo, mediaFormat, format, /* surface= */ null, crypto, loudnessCodecController);
}

/**
Expand All @@ -76,7 +79,8 @@ public static Configuration createForVideoDecoding(
Format format,
@Nullable Surface surface,
@Nullable MediaCrypto crypto) {
return new Configuration(codecInfo, mediaFormat, format, surface, crypto);
return new Configuration(
codecInfo, mediaFormat, format, surface, crypto, /* loudnessCodecController= */ null);
}

/** Information about the {@link MediaCodec} being configured. */
Expand All @@ -98,17 +102,22 @@ public static Configuration createForVideoDecoding(
/** For DRM protected playbacks, a {@link MediaCrypto} to use for decryption. */
@Nullable public final MediaCrypto crypto;

/** The {@link LoudnessCodecController} for audio codecs. */
@Nullable public final LoudnessCodecController loudnessCodecController;

private Configuration(
MediaCodecInfo codecInfo,
MediaFormat mediaFormat,
Format format,
@Nullable Surface surface,
@Nullable MediaCrypto crypto) {
@Nullable MediaCrypto crypto,
@Nullable LoudnessCodecController loudnessCodecController) {
this.codecInfo = codecInfo;
this.mediaFormat = mediaFormat;
this.format = format;
this.surface = surface;
this.crypto = crypto;
this.loudnessCodecController = loudnessCodecController;
}
}

Expand Down
Loading

0 comments on commit f7a726b

Please sign in to comment.