diff --git a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java index b83620df383..a8a19aee8a8 100644 --- a/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java +++ b/libraries/common/src/main/java/androidx/media3/common/util/CodecSpecificDataUtil.java @@ -31,6 +31,13 @@ public final class CodecSpecificDataUtil { private static final String[] HEVC_GENERAL_PROFILE_SPACE_STRINGS = new String[] {"", "A", "B", "C"}; + // MP4V-ES + private static final int VISUAL_OBJECT_LAYER = 1; + private static final int VISUAL_OBJECT_LAYER_START = 0x20; + private static final int EXTENDED_PAR = 0x0F; + private static final int RECTANGULAR = 0x00; + private static final int FINE_GRANULARITY_SCALABLE = 0x12; + /** * Parses an ALAC AudioSpecificConfig (i.e. an ALACSpecificConfig). @@ -72,6 +79,85 @@ public static boolean parseCea708InitializationData(List initializationD && initializationData.get(0)[0] == 1; } + /** + * Parses an MPEG-4 Visual configuration information, as defined in ISO/IEC14496-2 + * + * @param videoSpecificConfig A byte array containing the MPEG-4 Visual configuration information + * to parse. + * @return A pair consisting of the width and the height. + */ + public static Pair parseMpeg4VideoSpecificConfig(byte[] videoSpecificConfig) { + int offset = 0; + boolean foundVOL = false; + ParsableByteArray scdScratchBytes = new ParsableByteArray(videoSpecificConfig); + while (offset + 3 < videoSpecificConfig.length) { + if (scdScratchBytes.readUnsignedInt24() != VISUAL_OBJECT_LAYER + || (videoSpecificConfig[offset + 3] & 0xf0) != VISUAL_OBJECT_LAYER_START) { + scdScratchBytes.setPosition(scdScratchBytes.getPosition() - 2); + offset++; + continue; + } + foundVOL = true; + break; + } + + Assertions.checkArgument(foundVOL); + + ParsableBitArray scdScratchBits = new ParsableBitArray(videoSpecificConfig); + scdScratchBits.skipBits((offset + 4) * 8); + scdScratchBits.skipBits(1); // random_accessible_vol + + int videoObjectTypeIndication = scdScratchBits.readBits(8); + Assertions.checkArgument(videoObjectTypeIndication != FINE_GRANULARITY_SCALABLE); + + if (scdScratchBits.readBit()) { // object_layer_identifier + scdScratchBits.skipBits(4); // video_object_layer_verid + scdScratchBits.skipBits(3); // video_object_layer_priority + } + + int aspectRatioInfo = scdScratchBits.readBits(4); + if (aspectRatioInfo == EXTENDED_PAR) { + scdScratchBits.skipBits(8); // par_width + scdScratchBits.skipBits(8); // par_height + } + + if (scdScratchBits.readBit()) { // vol_control_parameters + scdScratchBits.skipBits(2); // chroma_format + scdScratchBits.skipBits(1); // low_delay + if (scdScratchBits.readBit()) { // vbv_parameters + scdScratchBits.skipBits(79); + } + } + + int videoObjectLayerShape = scdScratchBits.readBits(2); + Assertions.checkArgument(videoObjectLayerShape == RECTANGULAR); + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int vopTimeIncrementResolution = scdScratchBits.readBits(16); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + if (scdScratchBits.readBit()) { // fixed_vop_rate + Assertions.checkArgument(vopTimeIncrementResolution > 0); + --vopTimeIncrementResolution; + int numBits = 0; + while (vopTimeIncrementResolution > 0) { + ++numBits; + vopTimeIncrementResolution >>= 1; + } + scdScratchBits.skipBits(numBits); // fixed_vop_time_increment + } + + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerWidth = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + int videoObjectLayerHeight = scdScratchBits.readBits(13); + Assertions.checkArgument(scdScratchBits.readBit()); // marker_bit + + scdScratchBits.skipBits(1); // interlaced + + return Pair.create(videoObjectLayerWidth, videoObjectLayerHeight); + } + /** * Builds an RFC 6381 AVC codec string using the provided parameters. * diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 4c4521e682e..f44c68ba0a8 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -38,6 +38,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_AC3 = "AC3"; private static final String RTP_MEDIA_MPEG4_GENERIC = "MPEG4-GENERIC"; + private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; private static final String RTP_MEDIA_H264 = "H264"; /** Returns whether the format of a {@link MediaDescription} is supported. */ @@ -45,6 +46,7 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) { switch (Ascii.toUpperCase(mediaDescription.rtpMapAttribute.mediaEncoding)) { case RTP_MEDIA_AC3: case RTP_MEDIA_H264: + case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_MPEG4_GENERIC: return true; default: @@ -65,6 +67,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.AUDIO_AC3; case RTP_MEDIA_H264: return MimeTypes.VIDEO_H264; + case RTP_MEDIA_MPEG4_VIDEO: + return MimeTypes.VIDEO_MP4V; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; default: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index 5b6b9a46077..02733aadcca 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -25,6 +25,7 @@ import android.net.Uri; import android.util.Base64; +import android.util.Pair; import androidx.annotation.Nullable; import androidx.annotation.VisibleForTesting; import androidx.media3.common.C; @@ -44,10 +45,14 @@ // Format specific parameter names. private static final String PARAMETER_PROFILE_LEVEL_ID = "profile-level-id"; private static final String PARAMETER_SPROP_PARAMS = "sprop-parameter-sets"; + private static final String PARAMETER_CONFIG = "config"; + /** Prefix for the RFC6381 codecs string for AAC formats. */ private static final String AAC_CODECS_PREFIX = "mp4a.40."; /** Prefix for the RFC6381 codecs string for AVC formats. */ private static final String H264_CODECS_PREFIX = "avc1."; + /** Prefix for the RFC6416 codecs string for MPEG4V-ES formats. */ + private static final String MPEG4_CODECS_PREFIX = "mp4v"; private static final String GENERIC_CONTROL_ATTR = "*"; @@ -116,6 +121,10 @@ public int hashCode() { checkArgument(!fmtpParameters.isEmpty()); processAacFmtpAttribute(formatBuilder, fmtpParameters, channelCount, clockRate); break; + case MimeTypes.VIDEO_MP4V: + checkArgument(!fmtpParameters.isEmpty()); + processMPEG4FmtpAttribute(formatBuilder, fmtpParameters); + break; case MimeTypes.VIDEO_H264: checkArgument(!fmtpParameters.isEmpty()); processH264FmtpAttribute(formatBuilder, fmtpParameters); @@ -160,6 +169,24 @@ private static void processAacFmtpAttribute( AacUtil.buildAacLcAudioSpecificConfig(sampleRate, channelCount))); } + private static void processMPEG4FmtpAttribute( + Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { + @Nullable String configInput = fmtpAttributes.get(PARAMETER_CONFIG); + if (configInput != null) { + byte[] csd = Util.getBytesFromHexString(configInput); + ImmutableList initializationData = ImmutableList.of(csd); + formatBuilder.setInitializationData(initializationData); + Pair dimensions = CodecSpecificDataUtil.parseMpeg4VideoSpecificConfig(csd); + formatBuilder.setWidth(dimensions.first); + formatBuilder.setHeight(dimensions.second); + } + @Nullable String profileLevel = fmtpAttributes.get(PARAMETER_PROFILE_LEVEL_ID); + if (profileLevel == null) { + profileLevel = "1"; // default + } + formatBuilder.setCodecs(MPEG4_CODECS_PREFIX + profileLevel); + } + private static void processH264FmtpAttribute( Format.Builder formatBuilder, ImmutableMap fmtpAttributes) { checkArgument(fmtpAttributes.containsKey(PARAMETER_SPROP_PARAMS)); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 8fe084c1319..0d58957a341 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -38,6 +38,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { return new RtpAacReader(payloadFormat); case MimeTypes.VIDEO_H264: return new RtpH264Reader(payloadFormat); + case MimeTypes.VIDEO_MP4V: + return new RtpMPEG4Reader(payloadFormat); default: // No supported reader, returning null. } diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java new file mode 100644 index 00000000000..8d22cd82f38 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpMPEG4Reader.java @@ -0,0 +1,166 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.common.util.Util.castNonNull; + +import androidx.media3.common.C; +import androidx.media3.common.ParserException; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.TrackOutput; +import com.google.common.primitives.Bytes; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an H265 byte stream carried on RTP packets, and extracts H265 Access Units. Refer to + * RFC6416 for more details. + */ +/* package */ final class RtpMPEG4Reader implements RtpPayloadReader { + private static final String TAG = "RtpMPEG4Reader"; + + private static final long MEDIA_CLOCK_FREQUENCY = 90_000; + + /** + * VOP unit type. + */ + private static final int I_VOP = 0; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + @C.BufferFlags private int bufferFlags; + + private long firstReceivedTimestamp; + + private int previousSequenceNumber; + + private long startTimeOffsetUs; + + private int sampleLength; + + File output = null; + + FileOutputStream outputStream = null; + + /** Creates an instance. */ + public RtpMPEG4Reader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + firstReceivedTimestamp = C.TIME_UNSET; + previousSequenceNumber = C.INDEX_UNSET; + sampleLength = 0; + try { + output = new File("/data/local/tmp/" + "mpeg4v_es.out"); + outputStream = new FileOutputStream(output); + } catch (IOException e) { + //do nothing; + } + } + + private static long toSampleUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + (rtpTimestamp - firstReceivedRtpTimestamp), + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_VIDEO); + castNonNull(trackOutput).format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + Log.i(TAG, "RtpMPEG4Reader onReceivingFirstPacket"); + } + + @Override + public void consume(ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) + throws ParserException { + if (previousSequenceNumber != C.INDEX_UNSET && sequenceNumber != (previousSequenceNumber + 1)) { + Log.e(TAG, "Packet loss"); + } + checkStateNotNull(trackOutput); + + int limit = data.bytesLeft(); + trackOutput.sampleData(data, limit); + sampleLength += limit; + parseVopType(data); + + // Write the video sample + if (outputStream != null) { + try { + outputStream.write(data.getData()); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Marker (M) bit: The marker bit is set to 1 to indicate the last RTP + // packet(or only RTP packet) of a VOP. When multiple VOPs are carried + // in the same RTP packet, the marker bit is set to 1. + if (rtpMarker) { + if (firstReceivedTimestamp == C.TIME_UNSET) { + firstReceivedTimestamp = timestamp; + } + + long timeUs = toSampleUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata(timeUs, bufferFlags, sampleLength, 0, null); + sampleLength = 0; + } + + previousSequenceNumber = sequenceNumber; + } + + /** + * Parses VOP Coding type + * + * Sets {@link #bufferFlags} according to the VOP Coding type. + */ + private void parseVopType(ParsableByteArray data) { + // search for VOP_START_CODE (00 00 01 B6) + byte[] inputData = data.getData(); + byte[] startCode = {0x0, 0x0, 0x01, (byte) 0xB6}; + int vopStartCodePos = Bytes.indexOf(inputData, startCode); + if (vopStartCodePos != -1) { + data.setPosition(vopStartCodePos + 4); + int vopType = data.peekUnsignedByte() >> 6; + bufferFlags = getBufferFlagsFromVopType(vopType); + } + } + + @C.BufferFlags + private static int getBufferFlagsFromVopType(int vopType) { + return vopType == I_VOP ? C.BUFFER_FLAG_KEY_FRAME : 0; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + sampleLength = 0; + } +}