Skip to content

Commit

Permalink
Move muxing inside sample pipelines
Browse files Browse the repository at this point in the history
This logic is currently in the player renderers. With multi-asset, the
renderers will go into the AssetLoader, which shouldn't be responsible
for muxing.

PiperOrigin-RevId: 486860502
  • Loading branch information
kim-vde authored and microkatz committed Nov 8, 2022
1 parent b10b4e6 commit d8754b6
Show file tree
Hide file tree
Showing 8 changed files with 185 additions and 117 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,14 +28,13 @@
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.Util;
import java.nio.ByteBuffer;
import java.util.List;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.checkerframework.dataflow.qual.Pure;

/**
* Pipeline to decode audio samples, apply transformations on the raw samples, and re-encode them.
*/
/* package */ final class AudioTranscodingSamplePipeline implements SamplePipeline {
/* package */ final class AudioTranscodingSamplePipeline extends BaseSamplePipeline {

private static final int DEFAULT_ENCODER_BITRATE = 128 * 1024;

Expand All @@ -57,12 +56,15 @@
public AudioTranscodingSamplePipeline(
Format inputFormat,
long streamOffsetUs,
long streamStartPositionUs,
TransformationRequest transformationRequest,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
MuxerWrapper muxerWrapper,
FallbackListener fallbackListener)
throws TransformationException {
super(C.TRACK_TYPE_AUDIO, streamStartPositionUs, muxerWrapper);

decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderInputBuffer =
Expand Down Expand Up @@ -104,7 +106,9 @@ public AudioTranscodingSamplePipeline(
.setChannelCount(encoderInputAudioFormat.channelCount)
.setAverageBitrate(DEFAULT_ENCODER_BITRATE)
.build();
encoder = encoderFactory.createForAudioEncoding(requestedOutputFormat, allowedOutputMimeTypes);
encoder =
encoderFactory.createForAudioEncoding(
requestedOutputFormat, muxerWrapper.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO));

fallbackListener.onTransformationRequestFinalized(
createFallbackTransformationRequest(
Expand All @@ -126,7 +130,16 @@ public void queueInputBuffer() throws TransformationException {
}

@Override
public boolean processData() throws TransformationException {
public void release() {
if (speedChangingAudioProcessor != null) {
speedChangingAudioProcessor.reset();
}
decoder.release();
encoder.release();
}

@Override
protected boolean processDataUpToMuxer() throws TransformationException {
if (speedChangingAudioProcessor != null) {
return feedEncoderFromProcessor() || feedProcessorFromDecoder();
} else {
Expand All @@ -136,13 +149,13 @@ public boolean processData() throws TransformationException {

@Override
@Nullable
public Format getOutputFormat() throws TransformationException {
protected Format getMuxerInputFormat() throws TransformationException {
return encoder.getOutputFormat();
}

@Override
@Nullable
public DecoderInputBuffer getOutputBuffer() throws TransformationException {
protected DecoderInputBuffer getMuxerInputBuffer() throws TransformationException {
encoderOutputBuffer.data = encoder.getOutputBuffer();
if (encoderOutputBuffer.data == null) {
return null;
Expand All @@ -153,24 +166,15 @@ public DecoderInputBuffer getOutputBuffer() throws TransformationException {
}

@Override
public void releaseOutputBuffer() throws TransformationException {
protected void releaseMuxerInputBuffer() throws TransformationException {
encoder.releaseOutputBuffer(/* render= */ false);
}

@Override
public boolean isEnded() {
protected boolean isMuxerInputEnded() {
return encoder.isEnded();
}

@Override
public void release() {
if (speedChangingAudioProcessor != null) {
speedChangingAudioProcessor.reset();
}
decoder.release();
encoder.release();
}

/**
* Attempts to pass decoder output data to the encoder, and returns whether it may be possible to
* pass more data immediately by calling this method again.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.google.android.exoplayer2.transformer;

import static com.google.android.exoplayer2.util.Assertions.checkStateNotNull;

import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;

/* package */ abstract class BaseSamplePipeline implements SamplePipeline {

private final int trackType;
private final long streamStartPositionUs;
private final MuxerWrapper muxerWrapper;

private boolean muxerWrapperTrackAdded;
private boolean isEnded;

public BaseSamplePipeline(int trackType, long streamStartPositionUs, MuxerWrapper muxerWrapper) {
this.trackType = trackType;
this.streamStartPositionUs = streamStartPositionUs;
this.muxerWrapper = muxerWrapper;
}

@Override
public boolean processData() throws TransformationException {
return feedMuxer() || processDataUpToMuxer();
}

@Override
public boolean isEnded() {
return isEnded;
}

protected abstract boolean processDataUpToMuxer() throws TransformationException;

@Nullable
protected abstract Format getMuxerInputFormat() throws TransformationException;

@Nullable
protected abstract DecoderInputBuffer getMuxerInputBuffer() throws TransformationException;

protected abstract void releaseMuxerInputBuffer() throws TransformationException;

protected abstract boolean isMuxerInputEnded();

/**
* Attempts to pass encoded data to the muxer, and returns whether it may be possible to pass more
* data immediately by calling this method again.
*/
private boolean feedMuxer() throws TransformationException {
if (!muxerWrapperTrackAdded) {
@Nullable Format inputFormat = getMuxerInputFormat();
if (inputFormat == null) {
return false;
}
try {
muxerWrapper.addTrackFormat(inputFormat);
} catch (Muxer.MuxerException e) {
throw TransformationException.createForMuxer(
e, TransformationException.ERROR_CODE_MUXING_FAILED);
}
muxerWrapperTrackAdded = true;
}

if (isMuxerInputEnded()) {
muxerWrapper.endTrack(trackType);
isEnded = true;
return false;
}

@Nullable DecoderInputBuffer muxerInputBuffer = getMuxerInputBuffer();
if (muxerInputBuffer == null) {
return false;
}

long samplePresentationTimeUs = muxerInputBuffer.timeUs - streamStartPositionUs;
// TODO(b/204892224): Consider subtracting the first sample timestamp from the sample pipeline
// buffer from all samples so that they are guaranteed to start from zero in the output file.
try {
if (!muxerWrapper.writeSample(
trackType,
checkStateNotNull(muxerInputBuffer.data),
muxerInputBuffer.isKeyFrame(),
samplePresentationTimeUs)) {
return false;
}
} catch (Muxer.MuxerException e) {
throw TransformationException.createForMuxer(
e, TransformationException.ERROR_CODE_MUXING_FAILED);
}

releaseMuxerInputBuffer();
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,10 @@
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
import com.google.android.exoplayer2.util.MimeTypes;

/** Pipeline that passes through the samples without any re-encoding or transformation. */
/* package */ final class PassthroughSamplePipeline implements SamplePipeline {
/* package */ final class PassthroughSamplePipeline extends BaseSamplePipeline {

private final DecoderInputBuffer buffer;
private final Format format;
Expand All @@ -30,8 +31,11 @@

public PassthroughSamplePipeline(
Format format,
long streamStartPositionUs,
TransformationRequest transformationRequest,
MuxerWrapper muxerWrapper,
FallbackListener fallbackListener) {
super(MimeTypes.getTrackType(format.sampleMimeType), streamStartPositionUs, muxerWrapper);
this.format = format;
buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
hasPendingBuffer = false;
Expand All @@ -46,36 +50,38 @@ public DecoderInputBuffer dequeueInputBuffer() {

@Override
public void queueInputBuffer() {
hasPendingBuffer = true;
if (buffer.data != null && buffer.data.hasRemaining()) {
hasPendingBuffer = true;
}
}

@Override
public boolean processData() {
public void release() {}

@Override
protected boolean processDataUpToMuxer() {
return false;
}

@Override
public Format getOutputFormat() {
protected Format getMuxerInputFormat() {
return format;
}

@Override
@Nullable
public DecoderInputBuffer getOutputBuffer() {
protected DecoderInputBuffer getMuxerInputBuffer() {
return hasPendingBuffer ? buffer : null;
}

@Override
public void releaseOutputBuffer() {
protected void releaseMuxerInputBuffer() {
buffer.clear();
hasPendingBuffer = false;
}

@Override
public boolean isEnded() {
protected boolean isMuxerInputEnded() {
return buffer.isEndOfStream();
}

@Override
public void release() {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package com.google.android.exoplayer2.transformer;

import androidx.annotation.Nullable;
import com.google.android.exoplayer2.Format;
import com.google.android.exoplayer2.decoder.DecoderInputBuffer;

/**
Expand Down Expand Up @@ -45,21 +44,6 @@
*/
boolean processData() throws TransformationException;

/** Returns the output format of the pipeline if available, and {@code null} otherwise. */
@Nullable
Format getOutputFormat() throws TransformationException;

/** Returns an output buffer if the pipeline has produced output, and {@code null} otherwise */
@Nullable
DecoderInputBuffer getOutputBuffer() throws TransformationException;

/**
* Releases the pipeline's output buffer.
*
* <p>Should be called when the output buffer from {@link #getOutputBuffer()} is no longer needed.
*/
void releaseOutputBuffer() throws TransformationException;

/** Returns whether the pipeline has ended. */
boolean isEnded();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,16 +77,22 @@ protected boolean ensureConfigured() throws TransformationException {
Format inputFormat = checkNotNull(formatHolder.format);
if (shouldPassthrough(inputFormat)) {
samplePipeline =
new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
new PassthroughSamplePipeline(
inputFormat,
streamStartPositionUs,
transformationRequest,
muxerWrapper,
fallbackListener);
} else {
samplePipeline =
new AudioTranscodingSamplePipeline(
inputFormat,
streamOffsetUs,
streamStartPositionUs,
transformationRequest,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
muxerWrapper,
fallbackListener);
}
return true;
Expand Down
Loading

0 comments on commit d8754b6

Please sign in to comment.