diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 37132ebbf11..d04e3ae70d2 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -298,7 +298,6 @@ private static String buildCustomDiagnosticInfo(int errorCode) { private final DecoderInputBuffer buffer; private final DecoderInputBuffer bypassSampleBuffer; private final BatchBuffer bypassBatchBuffer; - private final TimedValueQueue formatQueue; private final ArrayList decodeOnlyPresentationTimestamps; private final MediaCodec.BufferInfo outputBufferInfo; private final ArrayDeque pendingOutputStreamChanges; @@ -359,6 +358,7 @@ private static String buildCustomDiagnosticInfo(int errorCode) { protected DecoderCounters decoderCounters; private OutputStreamInfo outputStreamInfo; private long lastProcessedOutputBufferTimeUs; + private boolean needToNotifyOutputFormatChangeAfterStreamChange; /** * @param trackType The {@link C.TrackType track type} that the renderer handles. @@ -385,7 +385,6 @@ public MediaCodecRenderer( buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); bypassSampleBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); bypassBatchBuffer = new BatchBuffer(); - formatQueue = new TimedValueQueue<>(); decodeOnlyPresentationTimestamps = new ArrayList<>(); outputBufferInfo = new MediaCodec.BufferInfo(); currentPlaybackSpeed = 1f; @@ -598,13 +597,15 @@ protected final void setPendingPlaybackException(ExoPlaybackException exception) protected final void updateOutputFormatForTime(long presentationTimeUs) throws ExoPlaybackException { boolean outputFormatChanged = false; - @Nullable Format format = formatQueue.pollFloor(presentationTimeUs); - if (format == null && codecOutputMediaFormatChanged) { - // If the codec's output MediaFormat has changed then there should be a corresponding Format - // change, which we've not found. Check the Format queue in case the corresponding - // presentation timestamp is greater than presentationTimeUs, which can happen for some codecs - // [Internal ref: b/162719047]. - format = formatQueue.pollFirst(); + @Nullable Format format = outputStreamInfo.formatQueue.pollFloor(presentationTimeUs); + if (format == null + && needToNotifyOutputFormatChangeAfterStreamChange + && codecOutputMediaFormat != null) { + // After a stream change or after the initial start, there should be an input format change, + // which we've not found. Check the Format queue in case the corresponding presentation + // timestamp is greater than presentationTimeUs, which can happen for some codecs + // [Internal ref: b/162719047 and https://github.com/google/ExoPlayer/issues/8594]. + format = outputStreamInfo.formatQueue.pollFirst(); } if (format != null) { outputFormat = format; @@ -613,6 +614,7 @@ protected final void updateOutputFormatForTime(long presentationTimeUs) if (outputFormatChanged || (codecOutputMediaFormatChanged && outputFormat != null)) { onOutputFormatChanged(outputFormat, codecOutputMediaFormat); codecOutputMediaFormatChanged = false; + needToNotifyOutputFormatChangeAfterStreamChange = false; } } @@ -669,10 +671,10 @@ protected void onPositionReset(long positionUs, boolean joining) throws ExoPlayb // If there is a format change on the input side still pending propagation to the output, we // need to queue a format next time a buffer is read. This is because we may not read a new // input format after the position reset. - if (formatQueue.size() > 0) { + if (outputStreamInfo.formatQueue.size() > 0) { waitingForFirstSampleInFormat = true; } - formatQueue.clear(); + outputStreamInfo.formatQueue.clear(); pendingOutputStreamChanges.clear(); } @@ -1336,7 +1338,11 @@ private boolean feedInputBuffer() throws ExoPlaybackException { decodeOnlyPresentationTimestamps.add(presentationTimeUs); } if (waitingForFirstSampleInFormat) { - formatQueue.add(presentationTimeUs, inputFormat); + if (!pendingOutputStreamChanges.isEmpty()) { + pendingOutputStreamChanges.peekLast().formatQueue.add(presentationTimeUs, inputFormat); + } else { + outputStreamInfo.formatQueue.add(presentationTimeUs, inputFormat); + } waitingForFirstSampleInFormat = false; } largestQueuedPresentationTimeUs = max(largestQueuedPresentationTimeUs, presentationTimeUs); @@ -2048,6 +2054,7 @@ protected final long getOutputStreamOffsetUs() { private void setOutputStreamInfo(OutputStreamInfo outputStreamInfo) { this.outputStreamInfo = outputStreamInfo; if (outputStreamInfo.streamOffsetUs != C.TIME_UNSET) { + needToNotifyOutputFormatChangeAfterStreamChange = true; onOutputStreamOffsetUsChanged(outputStreamInfo.streamOffsetUs); } } @@ -2513,12 +2520,14 @@ private static final class OutputStreamInfo { public final long previousStreamLastBufferTimeUs; public final long startPositionUs; public final long streamOffsetUs; + public final TimedValueQueue formatQueue; public OutputStreamInfo( long previousStreamLastBufferTimeUs, long startPositionUs, long streamOffsetUs) { this.previousStreamLastBufferTimeUs = previousStreamLastBufferTimeUs; this.startPositionUs = startPositionUs; this.streamOffsetUs = streamOffsetUs; + this.formatQueue = new TimedValueQueue<>(); } } diff --git a/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecRendererTest.java new file mode 100644 index 00000000000..cb247789094 --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/mediacodec/MediaCodecRendererTest.java @@ -0,0 +1,323 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.mediacodec; + +import static com.google.android.exoplayer2.decoder.DecoderReuseEvaluation.REUSE_RESULT_YES_WITHOUT_RECONFIGURATION; +import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static com.google.android.exoplayer2.testutil.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.inOrder; +import static org.mockito.Mockito.spy; + +import android.media.MediaCrypto; +import android.media.MediaFormat; +import android.os.SystemClock; +import androidx.annotation.Nullable; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.RendererConfiguration; +import com.google.android.exoplayer2.analytics.PlayerId; +import com.google.android.exoplayer2.decoder.DecoderReuseEvaluation; +import com.google.android.exoplayer2.drm.DrmSessionEventListener; +import com.google.android.exoplayer2.drm.DrmSessionManager; +import com.google.android.exoplayer2.testutil.FakeSampleStream; +import com.google.android.exoplayer2.upstream.DefaultAllocator; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InOrder; + +/** Unit tests for {@link MediaCodecRenderer} */ +@RunWith(AndroidJUnit4.class) +public class MediaCodecRendererTest { + + @Test + public void render_withReplaceStream_triggersOutputCallbacksInCorrectOrder() throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferBeyondDuration_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300, 400, 500, 600); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 400); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(400); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + @Test + public void + render_withReplaceStreamAndBufferLessThanStartPosition_triggersOutputCallbacksInCorrectOrder() + throws Exception { + Format format1 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1000).build(); + Format format2 = + new Format.Builder().setSampleMimeType(MimeTypes.AUDIO_AAC).setAverageBitrate(1500).build(); + FakeSampleStream fakeSampleStream1 = + createFakeSampleStream(format1, /* sampleTimesUs...= */ 0, 100, 200, 300); + FakeSampleStream fakeSampleStream2 = + createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200, 300, 400); + MediaCodecRenderer renderer = spy(new TestRenderer()); + renderer.init(/* index= */ 0, PlayerId.UNSET); + + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {format1}, + fakeSampleStream1, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.start(); + long positionUs = 0; + while (!renderer.hasReadStreamToEnd()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + renderer.replaceStream( + new Format[] {format2}, fakeSampleStream2, /* startPositionUs= */ 400, /* offsetUs= */ 200); + renderer.setCurrentStreamFinal(); + while (!renderer.isEnded()) { + renderer.render(positionUs, SystemClock.elapsedRealtime()); + positionUs += 100; + } + + InOrder inOrder = inOrder(renderer); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(0); + inOrder.verify(renderer).onOutputFormatChanged(eq(format1), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(0); + inOrder.verify(renderer).onProcessedOutputBuffer(100); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onOutputStreamOffsetUsChanged(200); + inOrder.verify(renderer).onProcessedStreamChange(); + inOrder.verify(renderer).onOutputFormatChanged(eq(format2), any()); + inOrder.verify(renderer).onProcessedOutputBuffer(200); + inOrder.verify(renderer).onProcessedOutputBuffer(300); + inOrder.verify(renderer).onProcessedOutputBuffer(400); + inOrder.verify(renderer).onProcessedOutputBuffer(500); + inOrder.verify(renderer).onProcessedOutputBuffer(600); + } + + private FakeSampleStream createFakeSampleStream(Format format, long... sampleTimesUs) { + ImmutableList.Builder sampleListBuilder = + ImmutableList.builder(); + for (long sampleTimeUs : sampleTimesUs) { + sampleListBuilder.add(oneByteSample(sampleTimeUs, C.BUFFER_FLAG_KEY_FRAME)); + } + sampleListBuilder.add(END_OF_STREAM_ITEM); + FakeSampleStream sampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + /* initialFormat= */ format, + sampleListBuilder.build()); + sampleStream.writeData(/* startPositionUs= */ 0); + return sampleStream; + } + + private static class TestRenderer extends MediaCodecRenderer { + + public TestRenderer() { + super( + C.TRACK_TYPE_AUDIO, + MediaCodecAdapter.Factory.DEFAULT, + /* mediaCodecSelector= */ (mimeType, requiresSecureDecoder, requiresTunnelingDecoder) -> + Collections.singletonList( + MediaCodecInfo.newInstance( + /* name= */ "name", + /* mimeType= */ mimeType, + /* codecMimeType= */ mimeType, + /* capabilities= */ null, + /* hardwareAccelerated= */ false, + /* softwareOnly= */ true, + /* vendor= */ false, + /* forceDisableAdaptive= */ false, + /* forceSecure= */ false)), + /* enableDecoderFallback= */ false, + /* assumedMinimumCodecOperatingRate= */ 44100); + } + + @Override + public String getName() { + return "test"; + } + + @Override + protected @Capabilities int supportsFormat(MediaCodecSelector mediaCodecSelector, Format format) + throws MediaCodecUtil.DecoderQueryException { + return RendererCapabilities.create(C.FORMAT_HANDLED); + } + + @Override + protected List getDecoderInfos( + MediaCodecSelector mediaCodecSelector, Format format, boolean requiresSecureDecoder) + throws MediaCodecUtil.DecoderQueryException { + return mediaCodecSelector.getDecoderInfos( + format.sampleMimeType, + /* requiresSecureDecoder= */ false, + /* requiresTunnelingDecoder= */ false); + } + + @Override + protected MediaCodecAdapter.Configuration getMediaCodecConfiguration( + MediaCodecInfo codecInfo, + Format format, + @Nullable MediaCrypto crypto, + float codecOperatingRate) { + return MediaCodecAdapter.Configuration.createForAudioDecoding( + codecInfo, new MediaFormat(), format, crypto); + } + + @Override + protected boolean processOutputBuffer( + long positionUs, + long elapsedRealtimeUs, + @Nullable MediaCodecAdapter codec, + @Nullable ByteBuffer buffer, + int bufferIndex, + int bufferFlags, + int sampleCount, + long bufferPresentationTimeUs, + boolean isDecodeOnlyBuffer, + boolean isLastBuffer, + Format format) + throws ExoPlaybackException { + if (bufferPresentationTimeUs <= positionUs) { + // Only release buffers when the position advances far enough for realistic behavior where + // input of buffers to the codec is faster than output. + codec.releaseOutputBuffer(bufferIndex, /* render= */ true); + return true; + } + return false; + } + + @Override + protected DecoderReuseEvaluation canReuseCodec( + MediaCodecInfo codecInfo, Format oldFormat, Format newFormat) { + return new DecoderReuseEvaluation( + codecInfo.name, + oldFormat, + newFormat, + REUSE_RESULT_YES_WITHOUT_RECONFIGURATION, + /* discardReasons= */ 0); + } + } +}