From ee02c6789a6a404a935fb4f23fdffa7d6bd62d9e Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Mon, 15 Oct 2018 19:44:39 -0700 Subject: [PATCH] Support seeking based on MLLT metadata Issue: #3241 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=217252254 --- RELEASENOTES.md | 3 + .../extractor/GaplessInfoHolder.java | 10 -- .../exoplayer2/extractor/mp3/MlltSeeker.java | 121 ++++++++++++++++++ .../extractor/mp3/Mp3Extractor.java | 41 +++++- .../exoplayer2/metadata/id3/Id3Decoder.java | 33 +++++ .../exoplayer2/metadata/id3/MlltFrame.java | 112 ++++++++++++++++ .../metadata/id3/MlltFrameTest.java | 49 +++++++ 7 files changed, 354 insertions(+), 15 deletions(-) create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/MlltSeeker.java create mode 100644 library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/MlltFrame.java create mode 100644 library/core/src/test/java/com/google/android/exoplayer2/metadata/id3/MlltFrameTest.java diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 01103a7a222..060b5f4dafd 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -21,6 +21,9 @@ ([#4788](https://github.com/google/ExoPlayer/issues/4788)). * SubRip: Add support for alignment tags, and remove tags from the displayed captions ([#4306](https://github.com/google/ExoPlayer/issues/4306)). +* Audio: + * Support seeking based on MLLT metadata + ([#3241](https://github.com/google/ExoPlayer/issues/3241)). * Fix issue where buffered position is not updated correctly when transitioning between periods ([#4899](https://github.com/google/ExoPlayer/issues/4899)). diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/GaplessInfoHolder.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/GaplessInfoHolder.java index 0742d96a06d..a0effc0df88 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/GaplessInfoHolder.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/GaplessInfoHolder.java @@ -18,7 +18,6 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.metadata.Metadata; import com.google.android.exoplayer2.metadata.id3.CommentFrame; -import com.google.android.exoplayer2.metadata.id3.Id3Decoder.FramePredicate; import com.google.android.exoplayer2.metadata.id3.InternalFrame; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,15 +27,6 @@ */ public final class GaplessInfoHolder { - /** - * A {@link FramePredicate} suitable for use when decoding {@link Metadata} that will be passed to - * {@link #setFromMetadata(Metadata)}. Only frames that might contain gapless playback information - * are decoded. - */ - public static final FramePredicate GAPLESS_INFO_ID3_FRAME_PREDICATE = - (majorVersion, id0, id1, id2, id3) -> - id0 == 'C' && id1 == 'O' && id2 == 'M' && (id3 == 'M' || majorVersion == 2); - private static final String GAPLESS_DOMAIN = "com.apple.iTunes"; private static final String GAPLESS_DESCRIPTION = "iTunSMPB"; private static final Pattern GAPLESS_COMMENT_PATTERN = diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/MlltSeeker.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/MlltSeeker.java new file mode 100644 index 00000000000..ff607b9482e --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/MlltSeeker.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.extractor.mp3; + +import android.util.Pair; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.extractor.SeekPoint; +import com.google.android.exoplayer2.metadata.id3.MlltFrame; +import com.google.android.exoplayer2.util.Util; + +/** MP3 seeker that uses metadata from an {@link MlltFrame}. */ +/* package */ final class MlltSeeker implements Mp3Extractor.Seeker { + + /** + * Returns an {@link MlltSeeker} for seeking in the stream. + * + * @param firstFramePosition The position of the start of the first frame in the stream. + * @param mlltFrame The MLLT frame with seeking metadata. + * @return An {@link MlltSeeker} for seeking in the stream. + */ + public static MlltSeeker create(long firstFramePosition, MlltFrame mlltFrame) { + int referenceCount = mlltFrame.bytesDeviations.length; + long[] referencePositions = new long[1 + referenceCount]; + long[] referenceTimesMs = new long[1 + referenceCount]; + referencePositions[0] = firstFramePosition; + referenceTimesMs[0] = 0; + long position = firstFramePosition; + long timeMs = 0; + for (int i = 1; i <= referenceCount; i++) { + position += mlltFrame.bytesBetweenReference + mlltFrame.bytesDeviations[i - 1]; + timeMs += mlltFrame.millisecondsBetweenReference + mlltFrame.millisecondsDeviations[i - 1]; + referencePositions[i] = position; + referenceTimesMs[i] = timeMs; + } + return new MlltSeeker(referencePositions, referenceTimesMs); + } + + private final long[] referencePositions; + private final long[] referenceTimesMs; + private final long durationUs; + + private MlltSeeker(long[] referencePositions, long[] referenceTimesMs) { + this.referencePositions = referencePositions; + this.referenceTimesMs = referenceTimesMs; + // Use the last reference point as the duration, as extrapolating variable bitrate at the end of + // the stream may give a large error. + durationUs = C.msToUs(referenceTimesMs[referenceTimesMs.length - 1]); + } + + @Override + public boolean isSeekable() { + return true; + } + + @Override + public SeekPoints getSeekPoints(long timeUs) { + timeUs = Util.constrainValue(timeUs, 0, durationUs); + Pair timeMsAndPosition = + linearlyInterpolate(C.usToMs(timeUs), referenceTimesMs, referencePositions); + timeUs = C.msToUs(timeMsAndPosition.first); + long position = timeMsAndPosition.second; + return new SeekPoints(new SeekPoint(timeUs, position)); + } + + @Override + public long getTimeUs(long position) { + Pair positionAndTimeMs = + linearlyInterpolate(position, referencePositions, referenceTimesMs); + return C.msToUs(positionAndTimeMs.second); + } + + @Override + public long getDurationUs() { + return durationUs; + } + + /** + * Given a set of reference points as coordinates in {@code xReferences} and {@code yReferences} + * and an x-axis value, linearly interpolates between corresponding reference points to give a + * y-axis value. + * + * @param x The x-axis value for which a y-axis value is needed. + * @param xReferences x coordinates of reference points. + * @param yReferences y coordinates of reference points. + * @return The linearly interpolated y-axis value. + */ + private static Pair linearlyInterpolate( + long x, long[] xReferences, long[] yReferences) { + int previousReferenceIndex = + Util.binarySearchFloor(xReferences, x, /* inclusive= */ true, /* stayInBounds= */ true); + long xPreviousReference = xReferences[previousReferenceIndex]; + long yPreviousReference = yReferences[previousReferenceIndex]; + int nextReferenceIndex = previousReferenceIndex + 1; + if (nextReferenceIndex == xReferences.length) { + return Pair.create(xPreviousReference, yPreviousReference); + } else { + long xNextReference = xReferences[nextReferenceIndex]; + long yNextReference = yReferences[nextReferenceIndex]; + double proportion = + xNextReference == xPreviousReference + ? 0.0 + : ((double) x - xPreviousReference) / (xNextReference - xPreviousReference); + long y = (long) (proportion * (yNextReference - yPreviousReference)) + yPreviousReference; + return Pair.create(x, y); + } + } + +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/Mp3Extractor.java b/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/Mp3Extractor.java index 92cf590a495..84f620734b8 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/Mp3Extractor.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/Mp3Extractor.java @@ -16,6 +16,7 @@ package com.google.android.exoplayer2.extractor.mp3; import android.support.annotation.IntDef; +import android.support.annotation.Nullable; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.ParserException; @@ -31,6 +32,8 @@ import com.google.android.exoplayer2.extractor.TrackOutput; import com.google.android.exoplayer2.metadata.Metadata; import com.google.android.exoplayer2.metadata.id3.Id3Decoder; +import com.google.android.exoplayer2.metadata.id3.Id3Decoder.FramePredicate; +import com.google.android.exoplayer2.metadata.id3.MlltFrame; import com.google.android.exoplayer2.util.ParsableByteArray; import com.google.android.exoplayer2.util.Util; import java.io.EOFException; @@ -68,6 +71,12 @@ public final class Mp3Extractor implements Extractor { */ public static final int FLAG_DISABLE_ID3_METADATA = 2; + /** Predicate that matches ID3 frames containing only required gapless/seeking metadata. */ + private static final FramePredicate REQUIRED_ID3_FRAME_PREDICATE = + (majorVersion, id0, id1, id2, id3) -> + ((id0 == 'C' && id1 == 'O' && id2 == 'M' && (id3 == 'M' || majorVersion == 2)) + || (id0 == 'M' && id1 == 'L' && id2 == 'L' && (id3 == 'T' || majorVersion == 2))); + /** * The maximum number of bytes to search when synchronizing, before giving up. */ @@ -174,7 +183,15 @@ public int read(ExtractorInput input, PositionHolder seekPosition) } } if (seeker == null) { - seeker = maybeReadSeekFrame(input); + // Read past any seek frame and set the seeker based on metadata or a seek frame. Metadata + // takes priority as it can provide greater precision. + Seeker seekFrameSeeker = maybeReadSeekFrame(input); + Seeker metadataSeeker = maybeHandleSeekMetadata(metadata, input.getPosition()); + if (metadataSeeker != null) { + seeker = metadataSeeker; + } else if (seekFrameSeeker != null) { + seeker = seekFrameSeeker; + } if (seeker == null || (!seeker.isSeekable() && (flags & FLAG_ENABLE_CONSTANT_BITRATE_SEEKING) != 0)) { seeker = getConstantBitrateSeeker(input); @@ -253,11 +270,11 @@ private boolean synchronize(ExtractorInput input, boolean sniffing) int searchLimitBytes = sniffing ? MAX_SNIFF_BYTES : MAX_SYNC_BYTES; input.resetPeekPosition(); if (input.getPosition() == 0) { - // We need to parse enough ID3 metadata to retrieve any gapless playback information even - // if ID3 metadata parsing is disabled. - boolean onlyDecodeGaplessInfoFrames = (flags & FLAG_DISABLE_ID3_METADATA) != 0; + // We need to parse enough ID3 metadata to retrieve any gapless/seeking playback information + // even if ID3 metadata parsing is disabled. + boolean parseAllId3Frames = (flags & FLAG_DISABLE_ID3_METADATA) == 0; Id3Decoder.FramePredicate id3FramePredicate = - onlyDecodeGaplessInfoFrames ? GaplessInfoHolder.GAPLESS_INFO_ID3_FRAME_PREDICATE : null; + parseAllId3Frames ? null : REQUIRED_ID3_FRAME_PREDICATE; metadata = id3Peeker.peekId3Data(input, id3FramePredicate); if (metadata != null) { gaplessInfoHolder.setFromMetadata(metadata); @@ -401,6 +418,20 @@ private static int getSeekFrameHeader(ParsableByteArray frame, int xingBase) { return SEEK_HEADER_UNSET; } + @Nullable + private static MlltSeeker maybeHandleSeekMetadata(Metadata metadata, long firstFramePosition) { + if (metadata != null) { + int length = metadata.length(); + for (int i = 0; i < length; i++) { + Metadata.Entry entry = metadata.get(i); + if (entry instanceof MlltFrame) { + return MlltSeeker.create(firstFramePosition, (MlltFrame) entry); + } + } + } + return null; + } + /** * {@link SeekMap} that also allows mapping from position (byte offset) back to time, which can be * used to work out the new sample basis timestamp after seeking and resynchronization. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/Id3Decoder.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/Id3Decoder.java index 0bf9d3b2493..63bf30dd116 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/Id3Decoder.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/Id3Decoder.java @@ -21,6 +21,7 @@ import com.google.android.exoplayer2.metadata.MetadataDecoder; import com.google.android.exoplayer2.metadata.MetadataInputBuffer; import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.util.ParsableBitArray; import com.google.android.exoplayer2.util.ParsableByteArray; import com.google.android.exoplayer2.util.Util; import java.io.UnsupportedEncodingException; @@ -382,6 +383,8 @@ private static boolean validateFrames(ParsableByteArray id3Data, int majorVersio } else if (frameId0 == 'C' && frameId1 == 'T' && frameId2 == 'O' && frameId3 == 'C') { frame = decodeChapterTOCFrame(id3Data, frameSize, majorVersion, unsignedIntFrameSizeHack, frameHeaderSize, framePredicate); + } else if (frameId0 == 'M' && frameId1 == 'L' && frameId2 == 'L' && frameId3 == 'T') { + frame = decodeMlltFrame(id3Data, frameSize); } else { String id = getFrameId(majorVersion, frameId0, frameId1, frameId2, frameId3); frame = decodeBinaryFrame(id3Data, frameSize, id); @@ -662,6 +665,36 @@ private static ChapterTocFrame decodeChapterTOCFrame( return new ChapterTocFrame(elementId, isRoot, isOrdered, children, subFrameArray); } + private static MlltFrame decodeMlltFrame(ParsableByteArray id3Data, int frameSize) { + // See ID3v2.4.0 native frames subsection 4.6. + int mpegFramesBetweenReference = id3Data.readUnsignedShort(); + int bytesBetweenReference = id3Data.readUnsignedInt24(); + int millisecondsBetweenReference = id3Data.readUnsignedInt24(); + int bitsForBytesDeviation = id3Data.readUnsignedByte(); + int bitsForMillisecondsDeviation = id3Data.readUnsignedByte(); + + ParsableBitArray references = new ParsableBitArray(); + references.reset(id3Data); + int referencesBits = 8 * (frameSize - 10); + int bitsPerReference = bitsForBytesDeviation + bitsForMillisecondsDeviation; + int referencesCount = referencesBits / bitsPerReference; + int[] bytesDeviations = new int[referencesCount]; + int[] millisecondsDeviations = new int[referencesCount]; + for (int i = 0; i < referencesCount; i++) { + int bytesDeviation = references.readBits(bitsForBytesDeviation); + int millisecondsDeviation = references.readBits(bitsForMillisecondsDeviation); + bytesDeviations[i] = bytesDeviation; + millisecondsDeviations[i] = millisecondsDeviation; + } + + return new MlltFrame( + mpegFramesBetweenReference, + bytesBetweenReference, + millisecondsBetweenReference, + bytesDeviations, + millisecondsDeviations); + } + private static BinaryFrame decodeBinaryFrame(ParsableByteArray id3Data, int frameSize, String id) { byte[] frame = new byte[frameSize]; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/MlltFrame.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/MlltFrame.java new file mode 100644 index 00000000000..06a4dd9d2d8 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/MlltFrame.java @@ -0,0 +1,112 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.metadata.id3; + +import android.os.Parcel; +import android.support.annotation.Nullable; +import java.util.Arrays; + +/** MPEG location lookup table frame. */ +public final class MlltFrame extends Id3Frame { + + public static final String ID = "MLLT"; + + public final int mpegFramesBetweenReference; + public final int bytesBetweenReference; + public final int millisecondsBetweenReference; + public final int[] bytesDeviations; + public final int[] millisecondsDeviations; + + public MlltFrame( + int mpegFramesBetweenReference, + int bytesBetweenReference, + int millisecondsBetweenReference, + int[] bytesDeviations, + int[] millisecondsDeviations) { + super(ID); + this.mpegFramesBetweenReference = mpegFramesBetweenReference; + this.bytesBetweenReference = bytesBetweenReference; + this.millisecondsBetweenReference = millisecondsBetweenReference; + this.bytesDeviations = bytesDeviations; + this.millisecondsDeviations = millisecondsDeviations; + } + + /* package */ MlltFrame(Parcel in) { + super(ID); + this.mpegFramesBetweenReference = in.readInt(); + this.bytesBetweenReference = in.readInt(); + this.millisecondsBetweenReference = in.readInt(); + this.bytesDeviations = in.createIntArray(); + this.millisecondsDeviations = in.createIntArray(); + } + + @Override + public boolean equals(@Nullable Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + MlltFrame other = (MlltFrame) obj; + return mpegFramesBetweenReference == other.mpegFramesBetweenReference + && bytesBetweenReference == other.bytesBetweenReference + && millisecondsBetweenReference == other.millisecondsBetweenReference + && Arrays.equals(bytesDeviations, other.bytesDeviations) + && Arrays.equals(millisecondsDeviations, other.millisecondsDeviations); + } + + @Override + public int hashCode() { + int result = 17; + result = 31 * result + mpegFramesBetweenReference; + result = 31 * result + bytesBetweenReference; + result = 31 * result + millisecondsBetweenReference; + result = 31 * result + Arrays.hashCode(bytesDeviations); + result = 31 * result + Arrays.hashCode(millisecondsDeviations); + return result; + } + + // Parcelable implementation. + + @Override + public void writeToParcel(Parcel dest, int flags) { + dest.writeInt(mpegFramesBetweenReference); + dest.writeInt(bytesBetweenReference); + dest.writeInt(millisecondsBetweenReference); + dest.writeIntArray(bytesDeviations); + dest.writeIntArray(millisecondsDeviations); + } + + @Override + public int describeContents() { + return 0; + } + + public static final Creator CREATOR = + new Creator() { + + @Override + public MlltFrame createFromParcel(Parcel in) { + return new MlltFrame(in); + } + + @Override + public MlltFrame[] newArray(int size) { + return new MlltFrame[size]; + } + }; +} diff --git a/library/core/src/test/java/com/google/android/exoplayer2/metadata/id3/MlltFrameTest.java b/library/core/src/test/java/com/google/android/exoplayer2/metadata/id3/MlltFrameTest.java new file mode 100644 index 00000000000..3e6520becaf --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/metadata/id3/MlltFrameTest.java @@ -0,0 +1,49 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.metadata.id3; + +import static com.google.common.truth.Truth.assertThat; + +import android.os.Parcel; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +/** Test for {@link MlltFrame}. */ +@RunWith(RobolectricTestRunner.class) +public final class MlltFrameTest { + + @Test + public void testParcelable() { + MlltFrame mlltFrameToParcel = + new MlltFrame( + /* mpegFramesBetweenReference= */ 1, + /* bytesBetweenReference= */ 1, + /* millisecondsBetweenReference= */ 1, + /* bytesDeviations= */ new int[] {1, 2}, + /* millisecondsDeviations= */ new int[] {1, 2}); + + Parcel parcel = Parcel.obtain(); + mlltFrameToParcel.writeToParcel(parcel, 0); + parcel.setDataPosition(0); + + MlltFrame mlltFrameFromParcel = MlltFrame.CREATOR.createFromParcel(parcel); + assertThat(mlltFrameFromParcel).isEqualTo(mlltFrameToParcel); + + parcel.recycle(); + } + +}