From ef568d2061132524eeefd08d4cb2f76c3b231b48 Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 10 Nov 2022 10:32:35 +0000 Subject: [PATCH] Add TransformerInternal The player is still driving the transformation at this point. The transformer thread will be added in another CL. PiperOrigin-RevId: 487479148 --- .../transformer/ExoPlayerAssetLoader.java | 104 +------ ...java => ExoPlayerAssetLoaderRenderer.java} | 86 +++--- .../exoplayer2/transformer/MuxerWrapper.java | 2 +- .../exoplayer2/transformer/Transformer.java | 30 +- .../transformer/TransformerAudioRenderer.java | 132 --------- .../transformer/TransformerInternal.java | 263 ++++++++++++++++++ .../transformer/TransformerVideoRenderer.java | 169 ----------- .../VideoTranscodingSamplePipeline.java | 6 +- 8 files changed, 345 insertions(+), 447 deletions(-) rename library/transformer/src/main/java/com/google/android/exoplayer2/transformer/{TransformerBaseRenderer.java => ExoPlayerAssetLoaderRenderer.java} (63%) delete mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java create mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java delete mode 100644 library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java index c0ce0d7336d..46e71b084c1 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoader.java @@ -34,6 +34,7 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.DefaultLoadControl; import com.google.android.exoplayer2.ExoPlayer; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.MediaItem; import com.google.android.exoplayer2.PlaybackException; import com.google.android.exoplayer2.Player; @@ -47,33 +48,28 @@ import com.google.android.exoplayer2.text.TextOutput; import com.google.android.exoplayer2.trackselection.DefaultTrackSelector; import com.google.android.exoplayer2.util.Clock; -import com.google.android.exoplayer2.util.DebugViewProvider; -import com.google.android.exoplayer2.util.Effect; -import com.google.android.exoplayer2.util.FrameProcessor; import com.google.android.exoplayer2.video.VideoRendererEventListener; -import com.google.common.collect.ImmutableList; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /* package */ final class ExoPlayerAssetLoader { public interface Listener { + void onTrackRegistered(); + + SamplePipeline onTrackAdded(Format format, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException; + void onEnded(); void onError(Exception e); } private final Context context; - private final TransformationRequest transformationRequest; - private final ImmutableList videoEffects; private final boolean removeAudio; private final boolean removeVideo; private final MediaSource.Factory mediaSourceFactory; - private final Codec.DecoderFactory decoderFactory; - private final Codec.EncoderFactory encoderFactory; - private final FrameProcessor.Factory frameProcessorFactory; private final Looper looper; - private final DebugViewProvider debugViewProvider; private final Clock clock; private @MonotonicNonNull MuxerWrapper muxerWrapper; @@ -82,28 +78,16 @@ public interface Listener { public ExoPlayerAssetLoader( Context context, - TransformationRequest transformationRequest, - ImmutableList videoEffects, boolean removeAudio, boolean removeVideo, MediaSource.Factory mediaSourceFactory, - Codec.DecoderFactory decoderFactory, - Codec.EncoderFactory encoderFactory, - FrameProcessor.Factory frameProcessorFactory, Looper looper, - DebugViewProvider debugViewProvider, Clock clock) { this.context = context; - this.transformationRequest = transformationRequest; - this.videoEffects = videoEffects; this.removeAudio = removeAudio; this.removeVideo = removeVideo; this.mediaSourceFactory = mediaSourceFactory; - this.decoderFactory = decoderFactory; - this.encoderFactory = encoderFactory; - this.frameProcessorFactory = frameProcessorFactory; this.looper = looper; - this.debugViewProvider = debugViewProvider; this.clock = clock; progressState = PROGRESS_STATE_NO_TRANSFORMATION; } @@ -112,7 +96,6 @@ public void start( MediaItem mediaItem, MuxerWrapper muxerWrapper, Listener listener, - FallbackListener fallbackListener, Transformer.AsyncErrorListener asyncErrorListener) { this.muxerWrapper = muxerWrapper; @@ -134,20 +117,7 @@ public void start( ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, - new RenderersFactoryImpl( - context, - muxerWrapper, - removeAudio, - removeVideo, - transformationRequest, - mediaItem.clippingConfiguration.startsAtKeyFrame, - videoEffects, - frameProcessorFactory, - encoderFactory, - decoderFactory, - fallbackListener, - asyncErrorListener, - debugViewProvider)) + new RenderersFactoryImpl(removeAudio, removeVideo, listener, asyncErrorListener)) .setMediaSourceFactory(mediaSourceFactory) .setTrackSelector(trackSelector) .setLoadControl(loadControl) @@ -187,48 +157,21 @@ public void release() { private static final class RenderersFactoryImpl implements RenderersFactory { - private final Context context; - private final MuxerWrapper muxerWrapper; private final TransformerMediaClock mediaClock; private final boolean removeAudio; private final boolean removeVideo; - private final TransformationRequest transformationRequest; - private final boolean clippingStartsAtKeyFrame; - private final ImmutableList videoEffects; - private final FrameProcessor.Factory frameProcessorFactory; - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final FallbackListener fallbackListener; + private final ExoPlayerAssetLoader.Listener assetLoaderListener; private final Transformer.AsyncErrorListener asyncErrorListener; - private final DebugViewProvider debugViewProvider; public RenderersFactoryImpl( - Context context, - MuxerWrapper muxerWrapper, boolean removeAudio, boolean removeVideo, - TransformationRequest transformationRequest, - boolean clippingStartsAtKeyFrame, - ImmutableList videoEffects, - FrameProcessor.Factory frameProcessorFactory, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - FallbackListener fallbackListener, - Transformer.AsyncErrorListener asyncErrorListener, - DebugViewProvider debugViewProvider) { - this.context = context; - this.muxerWrapper = muxerWrapper; + ExoPlayerAssetLoader.Listener assetLoaderListener, + Transformer.AsyncErrorListener asyncErrorListener) { this.removeAudio = removeAudio; this.removeVideo = removeVideo; - this.transformationRequest = transformationRequest; - this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; - this.videoEffects = videoEffects; - this.frameProcessorFactory = frameProcessorFactory; - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - this.fallbackListener = fallbackListener; + this.assetLoaderListener = assetLoaderListener; this.asyncErrorListener = asyncErrorListener; - this.debugViewProvider = debugViewProvider; mediaClock = new TransformerMediaClock(); } @@ -244,31 +187,14 @@ public Renderer[] createRenderers( int index = 0; if (!removeAudio) { renderers[index] = - new TransformerAudioRenderer( - muxerWrapper, - mediaClock, - transformationRequest, - encoderFactory, - decoderFactory, - asyncErrorListener, - fallbackListener); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_AUDIO, mediaClock, assetLoaderListener, asyncErrorListener); index++; } if (!removeVideo) { renderers[index] = - new TransformerVideoRenderer( - context, - muxerWrapper, - mediaClock, - transformationRequest, - clippingStartsAtKeyFrame, - videoEffects, - frameProcessorFactory, - encoderFactory, - decoderFactory, - asyncErrorListener, - fallbackListener, - debugViewProvider); + new ExoPlayerAssetLoaderRenderer( + C.TRACK_TYPE_VIDEO, mediaClock, assetLoaderListener, asyncErrorListener); index++; } return renderers; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java similarity index 63% rename from library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java rename to library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java index 3c08051aa26..42ca161d2a6 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerBaseRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/ExoPlayerAssetLoaderRenderer.java @@ -16,46 +16,53 @@ package com.google.android.exoplayer2.transformer; +import static com.google.android.exoplayer2.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED; +import static com.google.android.exoplayer2.source.SampleStream.FLAG_REQUIRE_FORMAT; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; + import androidx.annotation.Nullable; import com.google.android.exoplayer2.BaseRenderer; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.FormatHolder; import com.google.android.exoplayer2.RendererCapabilities; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; import com.google.android.exoplayer2.util.MediaClock; import com.google.android.exoplayer2.util.MimeTypes; -import com.google.errorprone.annotations.ForOverride; import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.checkerframework.checker.nullness.qual.RequiresNonNull; -/* package */ abstract class TransformerBaseRenderer extends BaseRenderer { +/* package */ final class ExoPlayerAssetLoaderRenderer extends BaseRenderer { + + private static final String TAG = "ExoPlayerAssetLoaderRenderer"; - protected final MuxerWrapper muxerWrapper; - protected final TransformerMediaClock mediaClock; - protected final TransformationRequest transformationRequest; - protected final Transformer.AsyncErrorListener asyncErrorListener; - protected final FallbackListener fallbackListener; + private final TransformerMediaClock mediaClock; + private final ExoPlayerAssetLoader.Listener assetLoaderListener; + private final Transformer.AsyncErrorListener asyncErrorListener; + private final DecoderInputBuffer decoderInputBuffer; private boolean isTransformationRunning; - protected long streamOffsetUs; - protected long streamStartPositionUs; - protected @MonotonicNonNull SamplePipeline samplePipeline; + private long streamOffsetUs; + private long streamStartPositionUs; + private @MonotonicNonNull SamplePipeline samplePipeline; - public TransformerBaseRenderer( + public ExoPlayerAssetLoaderRenderer( int trackType, - MuxerWrapper muxerWrapper, TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener) { + ExoPlayerAssetLoader.Listener assetLoaderListener, + Transformer.AsyncErrorListener asyncErrorListener) { super(trackType); - this.muxerWrapper = muxerWrapper; this.mediaClock = mediaClock; - this.transformationRequest = transformationRequest; + this.assetLoaderListener = assetLoaderListener; this.asyncErrorListener = asyncErrorListener; - this.fallbackListener = fallbackListener; + decoderInputBuffer = new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_DISABLED); + } + + @Override + public String getName() { + return TAG; } /** @@ -65,7 +72,7 @@ public TransformerBaseRenderer( * @return The {@link Capabilities} for this format. */ @Override - public final @Capabilities int supportsFormat(Format format) { + public @Capabilities int supportsFormat(Format format) { return RendererCapabilities.create( MimeTypes.getTrackType(format.sampleMimeType) == getTrackType() ? C.FORMAT_HANDLED @@ -73,22 +80,22 @@ public TransformerBaseRenderer( } @Override - public final MediaClock getMediaClock() { + public MediaClock getMediaClock() { return mediaClock; } @Override - public final boolean isReady() { + public boolean isReady() { return isSourceReady(); } @Override - public final boolean isEnded() { + public boolean isEnded() { return samplePipeline != null && samplePipeline.isEnded(); } @Override - public final void render(long positionUs, long elapsedRealtimeUs) { + public void render(long positionUs, long elapsedRealtimeUs) { try { if (!isTransformationRunning || isEnded() || !ensureConfigured()) { return; @@ -97,43 +104,56 @@ public final void render(long positionUs, long elapsedRealtimeUs) { while (samplePipeline.processData() || feedPipelineFromInput()) {} } catch (TransformationException e) { isTransformationRunning = false; - asyncErrorListener.onTransformationException(e); + asyncErrorListener.onTransformationError(e); } } @Override - protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { this.streamOffsetUs = offsetUs; this.streamStartPositionUs = startPositionUs; } @Override - protected final void onEnabled(boolean joining, boolean mayRenderStartOfStream) { - muxerWrapper.registerTrack(); - fallbackListener.registerTrack(); + protected void onEnabled(boolean joining, boolean mayRenderStartOfStream) { + assetLoaderListener.onTrackRegistered(); mediaClock.updateTimeForTrackType(getTrackType(), 0L); } @Override - protected final void onStarted() { + protected void onStarted() { isTransformationRunning = true; } @Override - protected final void onStopped() { + protected void onStopped() { isTransformationRunning = false; } @Override - protected final void onReset() { + protected void onReset() { if (samplePipeline != null) { samplePipeline.release(); } } - @ForOverride @EnsuresNonNullIf(expression = "samplePipeline", result = true) - protected abstract boolean ensureConfigured() throws TransformationException; + private boolean ensureConfigured() throws TransformationException { + if (samplePipeline != null) { + return true; + } + + FormatHolder formatHolder = getFormatHolder(); + @ReadDataResult + int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); + if (result != C.RESULT_FORMAT_READ) { + return false; + } + Format inputFormat = checkNotNull(formatHolder.format); + samplePipeline = + assetLoaderListener.onTrackAdded(inputFormat, streamStartPositionUs, streamOffsetUs); + return true; + } /** * Attempts to read input data and pass the input data to the sample pipeline. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java index 9e7aaf31d71..cde1f20de0f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java @@ -311,7 +311,7 @@ private void resetAbortTimer() { return; } isAborted = true; - asyncErrorListener.onTransformationException( + asyncErrorListener.onTransformationError( TransformationException.createForMuxer( new IllegalStateException( "No output sample written in the last " diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 0bb0209627e..1a1da5df297 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -31,7 +31,6 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.ExoPlayerLibraryInfo; import com.google.android.exoplayer2.MediaItem; -import com.google.android.exoplayer2.PlaybackException; import com.google.android.exoplayer2.effect.GlEffect; import com.google.android.exoplayer2.effect.GlEffectsFrameProcessor; import com.google.android.exoplayer2.effect.GlMatrixTransformation; @@ -550,7 +549,7 @@ default void onFallbackApplied( private final Looper looper; private final DebugViewProvider debugViewProvider; private final Clock clock; - private final ExoPlayerAssetLoader exoPlayerAssetLoader; + private final TransformerInternal transformerInternal; @Nullable private MuxerWrapper muxerWrapper; @Nullable private String outputPath; @@ -588,8 +587,8 @@ private Transformer( this.looper = looper; this.debugViewProvider = debugViewProvider; this.clock = clock; - exoPlayerAssetLoader = - new ExoPlayerAssetLoader( + transformerInternal = + new TransformerInternal( context, transformationRequest, videoEffects, @@ -728,7 +727,7 @@ private void startTransformationInternal(MediaItem mediaItem) { this.muxerWrapper = muxerWrapper; FallbackListener fallbackListener = new FallbackListener(mediaItem, listeners, transformationRequest); - exoPlayerAssetLoader.start( + transformerInternal.start( mediaItem, muxerWrapper, /* listener= */ componentListener, @@ -759,7 +758,7 @@ public Looper getApplicationLooper() { */ public @ProgressState int getProgress(ProgressHolder progressHolder) { verifyApplicationThread(); - return exoPlayerAssetLoader.getProgress(progressHolder); + return transformerInternal.getProgress(progressHolder); } /** @@ -789,7 +788,7 @@ public void cancel() { */ private void releaseResources(boolean forCancellation) throws TransformationException { transformationInProgress = false; - exoPlayerAssetLoader.release(); + transformerInternal.release(); if (muxerWrapper != null) { try { muxerWrapper.release(forCancellation); @@ -834,11 +833,11 @@ private long getCurrentOutputFileCurrentSizeBytes() { * *

Can be called from any thread. */ - void onTransformationException(TransformationException exception); + void onTransformationError(TransformationException exception); } private final class ComponentListener - implements ExoPlayerAssetLoader.Listener, AsyncErrorListener { + implements TransformerInternal.Listener, AsyncErrorListener { private final MediaItem mediaItem; private final Handler handler; @@ -849,21 +848,12 @@ public ComponentListener(MediaItem mediaItem, Looper looper) { } @Override - public void onError(Exception e) { - TransformationException transformationException = - e instanceof PlaybackException - ? TransformationException.createForPlaybackException((PlaybackException) e) - : TransformationException.createForUnexpected(e); - handleTransformationException(transformationException); - } - - @Override - public void onEnded() { + public void onTransformationCompleted() { handleTransformationEnded(/* exception= */ null); } @Override - public void onTransformationException(TransformationException exception) { + public void onTransformationError(TransformationException exception) { if (Looper.myLooper() == looper) { handleTransformationException(exception); } else { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java deleted file mode 100644 index b2614fad812..00000000000 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Copyright 2020 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.exoplayer2.transformer; - -import static com.google.android.exoplayer2.source.SampleStream.FLAG_REQUIRE_FORMAT; -import static com.google.android.exoplayer2.util.Assertions.checkNotNull; - -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.FormatHolder; -import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.metadata.Metadata; -import com.google.android.exoplayer2.metadata.mp4.SlowMotionData; -import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; - -/* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer { - - private static final String TAG = "TAudioRenderer"; - - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final DecoderInputBuffer decoderInputBuffer; - - public TransformerAudioRenderer( - MuxerWrapper muxerWrapper, - TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener) { - super( - C.TRACK_TYPE_AUDIO, - muxerWrapper, - mediaClock, - transformationRequest, - asyncErrorListener, - fallbackListener); - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - decoderInputBuffer = - new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - } - - @Override - public String getName() { - return TAG; - } - - /** Attempts to read the input format and to initialize the {@link SamplePipeline}. */ - @Override - protected boolean ensureConfigured() throws TransformationException { - if (samplePipeline != null) { - return true; - } - FormatHolder formatHolder = getFormatHolder(); - @ReadDataResult - int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; - } - Format inputFormat = checkNotNull(formatHolder.format); - if (shouldPassthrough(inputFormat)) { - samplePipeline = - new PassthroughSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - muxerWrapper, - fallbackListener); - } else { - samplePipeline = - new AudioTranscodingSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - decoderFactory, - encoderFactory, - muxerWrapper, - fallbackListener); - } - return true; - } - - private boolean shouldPassthrough(Format inputFormat) { - if (encoderFactory.audioNeedsEncoding()) { - return false; - } - if (transformationRequest.audioMimeType != null - && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { - return false; - } - if (transformationRequest.audioMimeType == null - && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { - return false; - } - if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { - return false; - } - return true; - } - - private static boolean isSlowMotion(Format format) { - @Nullable Metadata metadata = format.metadata; - if (metadata == null) { - return false; - } - for (int i = 0; i < metadata.length(); i++) { - if (metadata.get(i) instanceof SlowMotionData) { - return true; - } - } - return false; - } -} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java new file mode 100644 index 00000000000..3d0c03f12ad --- /dev/null +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerInternal.java @@ -0,0 +1,263 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.android.exoplayer2.transformer; + +import android.content.Context; +import android.os.Looper; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.MediaItem; +import com.google.android.exoplayer2.PlaybackException; +import com.google.android.exoplayer2.metadata.Metadata; +import com.google.android.exoplayer2.metadata.mp4.SlowMotionData; +import com.google.android.exoplayer2.source.MediaSource; +import com.google.android.exoplayer2.util.Clock; +import com.google.android.exoplayer2.util.DebugViewProvider; +import com.google.android.exoplayer2.util.Effect; +import com.google.android.exoplayer2.util.FrameProcessor; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.common.collect.ImmutableList; + +/* package */ final class TransformerInternal { + + public interface Listener { + + void onTransformationCompleted(); + + void onTransformationError(TransformationException exception); + } + + private final Context context; + private final TransformationRequest transformationRequest; + private final ImmutableList videoEffects; + private final Codec.DecoderFactory decoderFactory; + private final Codec.EncoderFactory encoderFactory; + private final FrameProcessor.Factory frameProcessorFactory; + private final DebugViewProvider debugViewProvider; + private final ExoPlayerAssetLoader exoPlayerAssetLoader; + + public TransformerInternal( + Context context, + TransformationRequest transformationRequest, + ImmutableList videoEffects, + boolean removeAudio, + boolean removeVideo, + MediaSource.Factory mediaSourceFactory, + Codec.DecoderFactory decoderFactory, + Codec.EncoderFactory encoderFactory, + FrameProcessor.Factory frameProcessorFactory, + Looper looper, + DebugViewProvider debugViewProvider, + Clock clock) { + this.context = context; + this.transformationRequest = transformationRequest; + this.videoEffects = videoEffects; + this.decoderFactory = decoderFactory; + this.encoderFactory = encoderFactory; + this.frameProcessorFactory = frameProcessorFactory; + this.debugViewProvider = debugViewProvider; + exoPlayerAssetLoader = + new ExoPlayerAssetLoader( + context, removeAudio, removeVideo, mediaSourceFactory, looper, clock); + } + + public void start( + MediaItem mediaItem, + MuxerWrapper muxerWrapper, + Listener listener, + FallbackListener fallbackListener, + Transformer.AsyncErrorListener asyncErrorListener) { + ComponentListener componentListener = + new ComponentListener( + mediaItem, muxerWrapper, listener, fallbackListener, asyncErrorListener); + exoPlayerAssetLoader.start(mediaItem, muxerWrapper, componentListener, asyncErrorListener); + } + + public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { + return exoPlayerAssetLoader.getProgress(progressHolder); + } + + public void release() { + exoPlayerAssetLoader.release(); + } + + private class ComponentListener implements ExoPlayerAssetLoader.Listener { + + private final MediaItem mediaItem; + private final MuxerWrapper muxerWrapper; + private final TransformerInternal.Listener listener; + private final FallbackListener fallbackListener; + private final Transformer.AsyncErrorListener asyncErrorListener; + + public ComponentListener( + MediaItem mediaItem, + MuxerWrapper muxerWrapper, + Listener listener, + FallbackListener fallbackListener, + Transformer.AsyncErrorListener asyncErrorListener) { + this.mediaItem = mediaItem; + this.muxerWrapper = muxerWrapper; + this.listener = listener; + this.fallbackListener = fallbackListener; + this.asyncErrorListener = asyncErrorListener; + } + + @Override + public void onTrackRegistered() { + muxerWrapper.registerTrack(); + fallbackListener.registerTrack(); + } + + @Override + public SamplePipeline onTrackAdded( + Format format, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException { + return getSamplePipeline(format, streamStartPositionUs, streamOffsetUs); + } + + @Override + public void onError(Exception e) { + TransformationException transformationException = + e instanceof PlaybackException + ? TransformationException.createForPlaybackException((PlaybackException) e) + : TransformationException.createForUnexpected(e); + listener.onTransformationError(transformationException); + } + + @Override + public void onEnded() { + listener.onTransformationCompleted(); + } + + private SamplePipeline getSamplePipeline( + Format inputFormat, long streamStartPositionUs, long streamOffsetUs) + throws TransformationException { + if (MimeTypes.isAudio(inputFormat.sampleMimeType) && shouldTranscodeAudio(inputFormat)) { + return new AudioTranscodingSamplePipeline( + inputFormat, + streamOffsetUs, + streamOffsetUs, + transformationRequest, + decoderFactory, + encoderFactory, + muxerWrapper, + fallbackListener); + } else if (MimeTypes.isVideo(inputFormat.sampleMimeType) + && shouldTranscodeVideo(inputFormat, streamStartPositionUs, streamOffsetUs)) { + return new VideoTranscodingSamplePipeline( + context, + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest, + videoEffects, + frameProcessorFactory, + decoderFactory, + encoderFactory, + muxerWrapper, + fallbackListener, + asyncErrorListener, + debugViewProvider); + } else { + return new PassthroughSamplePipeline( + inputFormat, + streamOffsetUs, + streamStartPositionUs, + transformationRequest, + muxerWrapper, + fallbackListener); + } + } + + private boolean shouldTranscodeAudio(Format inputFormat) { + if (encoderFactory.audioNeedsEncoding()) { + return true; + } + if (transformationRequest.audioMimeType != null + && !transformationRequest.audioMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.audioMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.flattenForSlowMotion && isSlowMotion(inputFormat)) { + return true; + } + return false; + } + + private boolean isSlowMotion(Format format) { + @Nullable Metadata metadata = format.metadata; + if (metadata == null) { + return false; + } + for (int i = 0; i < metadata.length(); i++) { + if (metadata.get(i) instanceof SlowMotionData) { + return true; + } + } + return false; + } + + private boolean shouldTranscodeVideo( + Format inputFormat, long streamStartPositionUs, long streamOffsetUs) { + if ((streamStartPositionUs - streamOffsetUs) != 0 + && !mediaItem.clippingConfiguration.startsAtKeyFrame) { + return true; + } + if (encoderFactory.videoNeedsEncoding()) { + return true; + } + if (transformationRequest.enableRequestSdrToneMapping) { + return true; + } + if (transformationRequest.videoMimeType != null + && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformationRequest.videoMimeType == null + && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { + return true; + } + if (inputFormat.pixelWidthHeightRatio != 1f) { + return true; + } + if (transformationRequest.rotationDegrees != 0f) { + return true; + } + if (transformationRequest.scaleX != 1f) { + return true; + } + if (transformationRequest.scaleY != 1f) { + return true; + } + // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. + int decodedHeight = + (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; + if (transformationRequest.outputHeight != C.LENGTH_UNSET + && transformationRequest.outputHeight != decodedHeight) { + return true; + } + if (!videoEffects.isEmpty()) { + return true; + } + return false; + } + } +} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java deleted file mode 100644 index 868007eb567..00000000000 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerVideoRenderer.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright 2021 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.google.android.exoplayer2.transformer; - -import static com.google.android.exoplayer2.source.SampleStream.FLAG_REQUIRE_FORMAT; -import static com.google.android.exoplayer2.util.Assertions.checkNotNull; - -import android.content.Context; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.FormatHolder; -import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; -import com.google.android.exoplayer2.util.DebugViewProvider; -import com.google.android.exoplayer2.util.Effect; -import com.google.android.exoplayer2.util.FrameProcessor; -import com.google.common.collect.ImmutableList; - -/* package */ final class TransformerVideoRenderer extends TransformerBaseRenderer { - - private static final String TAG = "TVideoRenderer"; - - private final Context context; - private final boolean clippingStartsAtKeyFrame; - private final ImmutableList effects; - private final FrameProcessor.Factory frameProcessorFactory; - private final Codec.EncoderFactory encoderFactory; - private final Codec.DecoderFactory decoderFactory; - private final DebugViewProvider debugViewProvider; - private final DecoderInputBuffer decoderInputBuffer; - - public TransformerVideoRenderer( - Context context, - MuxerWrapper muxerWrapper, - TransformerMediaClock mediaClock, - TransformationRequest transformationRequest, - boolean clippingStartsAtKeyFrame, - ImmutableList effects, - FrameProcessor.Factory frameProcessorFactory, - Codec.EncoderFactory encoderFactory, - Codec.DecoderFactory decoderFactory, - Transformer.AsyncErrorListener asyncErrorListener, - FallbackListener fallbackListener, - DebugViewProvider debugViewProvider) { - super( - C.TRACK_TYPE_VIDEO, - muxerWrapper, - mediaClock, - transformationRequest, - asyncErrorListener, - fallbackListener); - this.context = context; - this.clippingStartsAtKeyFrame = clippingStartsAtKeyFrame; - this.effects = effects; - this.frameProcessorFactory = frameProcessorFactory; - this.encoderFactory = encoderFactory; - this.decoderFactory = decoderFactory; - this.debugViewProvider = debugViewProvider; - decoderInputBuffer = - new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); - } - - @Override - public String getName() { - return TAG; - } - - /** Attempts to read the input format and to initialize the {@link SamplePipeline}. */ - @Override - protected boolean ensureConfigured() throws TransformationException { - if (samplePipeline != null) { - return true; - } - FormatHolder formatHolder = getFormatHolder(); - @ReadDataResult - int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT); - if (result != C.RESULT_FORMAT_READ) { - return false; - } - Format inputFormat = checkNotNull(formatHolder.format); - if (shouldTranscode(inputFormat)) { - samplePipeline = - new VideoTranscodingSamplePipeline( - context, - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - effects, - frameProcessorFactory, - decoderFactory, - encoderFactory, - muxerWrapper, - fallbackListener, - asyncErrorListener, - debugViewProvider); - } else { - samplePipeline = - new PassthroughSamplePipeline( - inputFormat, - streamOffsetUs, - streamStartPositionUs, - transformationRequest, - muxerWrapper, - fallbackListener); - } - return true; - } - - private boolean shouldTranscode(Format inputFormat) { - if ((streamStartPositionUs - streamOffsetUs) != 0 && !clippingStartsAtKeyFrame) { - return true; - } - if (encoderFactory.videoNeedsEncoding()) { - return true; - } - if (transformationRequest.enableRequestSdrToneMapping) { - return true; - } - if (transformationRequest.forceInterpretHdrVideoAsSdr) { - return true; - } - if (transformationRequest.videoMimeType != null - && !transformationRequest.videoMimeType.equals(inputFormat.sampleMimeType)) { - return true; - } - if (transformationRequest.videoMimeType == null - && !muxerWrapper.supportsSampleMimeType(inputFormat.sampleMimeType)) { - return true; - } - if (inputFormat.pixelWidthHeightRatio != 1f) { - return true; - } - if (transformationRequest.rotationDegrees != 0f) { - return true; - } - if (transformationRequest.scaleX != 1f) { - return true; - } - if (transformationRequest.scaleY != 1f) { - return true; - } - // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. - int decodedHeight = - (inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width; - if (transformationRequest.outputHeight != C.LENGTH_UNSET - && transformationRequest.outputHeight != decodedHeight) { - return true; - } - if (!effects.isEmpty()) { - return true; - } - return false; - } -} diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java index 8ae54d1b81c..5523fc058a4 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoTranscodingSamplePipeline.java @@ -154,7 +154,7 @@ public void onOutputSizeChanged(int width, int height) { checkNotNull(frameProcessor) .setOutputSurfaceInfo(encoderWrapper.getSurfaceInfo(width, height)); } catch (TransformationException exception) { - asyncErrorListener.onTransformationException(exception); + asyncErrorListener.onTransformationError(exception); } } @@ -165,7 +165,7 @@ public void onOutputFrameAvailable(long presentationTimeUs) { @Override public void onFrameProcessingError(FrameProcessingException exception) { - asyncErrorListener.onTransformationException( + asyncErrorListener.onTransformationError( TransformationException.createForFrameProcessingException( exception, TransformationException.ERROR_CODE_FRAME_PROCESSING_FAILED)); } @@ -175,7 +175,7 @@ public void onFrameProcessingEnded() { try { encoderWrapper.signalEndOfInputStream(); } catch (TransformationException exception) { - asyncErrorListener.onTransformationException(exception); + asyncErrorListener.onTransformationError(exception); } } },