Support CEA-608 closed-captions in H262/MPEG-TS.

library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/DefaultTsPayloadReaderFactory.java

@@ -104,7 +104,7 @@ public TsPayloadReader createPayloadReader(int streamType, EsInfo esInfo) {
       case TsExtractor.TS_STREAM_TYPE_HDMV_DTS:
         return new PesReader(new DtsReader(esInfo.language));
       case TsExtractor.TS_STREAM_TYPE_H262:
-        return new PesReader(new H262Reader());
+        return new PesReader(new H262Reader(buildSeiReader(esInfo)));
       case TsExtractor.TS_STREAM_TYPE_H264:
         return isSet(FLAG_IGNORE_H264_STREAM) ? null
             : new PesReader(new H264Reader(buildSeiReader(esInfo),

library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H262Reader.java

@@ -21,6 +21,7 @@
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.ts.TsPayloadReader.TrackIdGenerator;
+import com.google.android.exoplayer2.text.cea.CeaUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
 import com.google.android.exoplayer2.util.ParsableByteArray;
@@ -33,10 +34,13 @@
 public final class H262Reader implements ElementaryStreamReader {
 
   private static final int START_PICTURE = 0x00;
+  private static final int START_USER_DATA = 0xB2;
   private static final int START_SEQUENCE_HEADER = 0xB3;
   private static final int START_EXTENSION = 0xB5;
   private static final int START_GROUP = 0xB8;
 
+  private final SeiReader seiReader;
+
   private String formatId;
   private TrackOutput output;
 
@@ -53,25 +57,37 @@ public final class H262Reader implements ElementaryStreamReader {
   private final CsdBuffer csdBuffer;
   private boolean foundFirstFrameInGroup;
   private long totalBytesWritten;
+  private final UserDataBuffer userDataBuffer;
 
   // Per packet state that gets reset at the start of each packet.
   private long pesTimeUs;
   private boolean pesPtsUsAvailable;
 
+  // Scratch variables to avoid allocations.
+  private final ParsableByteArray userDataWrapper;
+
   // Per sample state that gets reset at the start of each frame.
   private boolean isKeyframe;
   private long framePosition;
   private long frameTimeUs;
 
-  public H262Reader() {
+  public H262Reader(SeiReader seiReader) {
+    this.seiReader = seiReader;
     prefixFlags = new boolean[4];
     csdBuffer = new CsdBuffer(128);
+    userDataBuffer = new UserDataBuffer(64);
+    userDataWrapper = new ParsableByteArray();
+  }
+
+  public H262Reader() {
+    this(null);
   }
 
   @Override
   public void seek() {
     NalUnitUtil.clearPrefixFlags(prefixFlags);
     csdBuffer.reset();
+    userDataBuffer.reset();
     pesPtsUsAvailable = false;
     foundFirstFrameInGroup = false;
     totalBytesWritten = 0;
@@ -82,6 +98,9 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
     idGenerator.generateNewId();
     formatId = idGenerator.getFormatId();
     output = extractorOutput.track(idGenerator.getTrackId(), C.TRACK_TYPE_VIDEO);
+    if (seiReader != null) {
+      seiReader.createTracks(extractorOutput, idGenerator);
+    }
   }
 
   @Override
@@ -110,6 +129,8 @@ public void consume(ParsableByteArray data) {
         // We've scanned to the end of the data without finding another start code.
         if (!hasOutputFormat) {
           csdBuffer.onData(dataArray, offset, limit);
+        } else if (seiReader != null) {
+          userDataBuffer.onData(dataArray, offset, limit);
         }
         return;
       }
@@ -136,6 +157,20 @@ public void consume(ParsableByteArray data) {
         }
       }
 
+      if (hasOutputFormat && seiReader != null) {
+        int lengthToStartCode = startCodeOffset - offset;
+        if (lengthToStartCode > 0) {
+          userDataBuffer.onData(dataArray, offset, startCodeOffset);
+        }
+        int bytesAlreadyPassed = lengthToStartCode < 0 ? -lengthToStartCode : 0;
+        if (userDataBuffer.onStartCode(startCodeValue, bytesAlreadyPassed)) {
+          userDataWrapper.reset(userDataBuffer.data, userDataBuffer.length);
+          userDataWrapper.setPosition(4);
+          seiReader.consume(frameTimeUs, userDataWrapper, CeaUtil.MODE_H262);
+          userDataBuffer.reset();
+        }
+      }
+
       if (hasOutputFormat && (startCodeValue == START_GROUP || startCodeValue == START_PICTURE)) {
         int bytesWrittenPastStartCode = limit - startCodeOffset;
         if (foundFirstFrameInGroup) {
@@ -286,4 +321,55 @@ public void onData(byte[] newData, int offset, int limit) {
 
   }
 
+  private static final class UserDataBuffer {
+
+    private boolean isFilling;
+
+    public int length;
+    public byte[] data;
+
+    public UserDataBuffer(int initialCapacity) {
+      data = new byte[initialCapacity];
+    }
+
+    /**
+     * Resets the buffer, clearing any data that it holds.
+     */
+    public void reset() {
+      isFilling = false;
+      length = 0;
+    }
+
+    /**
+     * Called to pass stream data.
+     *
+     * @param newData Holds the data being passed.
+     * @param offset The offset of the data in {@code data}.
+     * @param limit The limit (exclusive) of the data in {@code data}.
+     */
+    public void onData(byte[] newData, int offset, int limit) {
+      if (!isFilling) {
+        return;
+      }
+      int readLength = limit - offset;
+      if (data.length < length + readLength) {
+        data = Arrays.copyOf(data, (length + readLength) * 2);
+      }
+      System.arraycopy(newData, offset, data, length, readLength);
+      length += readLength;
+    }
+
+    public boolean onStartCode(int startCodeValue, int bytesAlreadyPassed) {
+      if (isFilling) {
+        length -= bytesAlreadyPassed;
+        isFilling = false;
+        return true;
+      } else if (startCodeValue == START_USER_DATA) {
+        isFilling = true;
+      }
+      return false;
+    }
+
+  }
+
 }

library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H264Reader.java

@@ -21,6 +21,7 @@
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.ts.TsPayloadReader.TrackIdGenerator;
+import com.google.android.exoplayer2.text.cea.CeaUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
 import com.google.android.exoplayer2.util.NalUnitUtil.SpsData;
@@ -203,7 +204,7 @@ private void endNalUnit(long position, int offset, int discardPadding, long pesT
       int unescapedLength = NalUnitUtil.unescapeStream(sei.nalData, sei.nalLength);
       seiWrapper.reset(sei.nalData, unescapedLength);
       seiWrapper.setPosition(4); // NAL prefix and nal_unit() header.
-      seiReader.consume(pesTimeUs, seiWrapper);
+      seiReader.consume(pesTimeUs, seiWrapper, CeaUtil.MODE_H264);
     }
     sampleReader.endNalUnit(position, offset);
   }

library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/H265Reader.java

@@ -21,6 +21,7 @@
 import com.google.android.exoplayer2.extractor.ExtractorOutput;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.extractor.ts.TsPayloadReader.TrackIdGenerator;
+import com.google.android.exoplayer2.text.cea.CeaUtil;
 import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.NalUnitUtil;
 import com.google.android.exoplayer2.util.ParsableByteArray;
@@ -201,15 +202,15 @@ private void endNalUnit(long position, int offset, int discardPadding, long pesT
 
       // Skip the NAL prefix and type.
       seiWrapper.skipBytes(5);
-      seiReader.consume(pesTimeUs, seiWrapper);
+      seiReader.consume(pesTimeUs, seiWrapper, CeaUtil.MODE_H265);
     }
     if (suffixSei.endNalUnit(discardPadding)) {
       int unescapedLength = NalUnitUtil.unescapeStream(suffixSei.nalData, suffixSei.nalLength);
       seiWrapper.reset(suffixSei.nalData, unescapedLength);
 
       // Skip the NAL prefix and type.
       seiWrapper.skipBytes(5);
-      seiReader.consume(pesTimeUs, seiWrapper);
+      seiReader.consume(pesTimeUs, seiWrapper, CeaUtil.MODE_H265);
     }
   }
 

library/core/src/main/java/com/google/android/exoplayer2/extractor/ts/SeiReader.java

@@ -59,8 +59,8 @@ public void createTracks(ExtractorOutput extractorOutput, TrackIdGenerator idGen
     }
   }
 
-  public void consume(long pesTimeUs, ParsableByteArray seiBuffer) {
-    CeaUtil.consume(pesTimeUs, seiBuffer, outputs);
+  public void consume(long pesTimeUs, ParsableByteArray seiBuffer, int mode) {
+    CeaUtil.consume(pesTimeUs, seiBuffer, outputs, mode);
   }
 
 }

library/core/src/main/java/com/google/android/exoplayer2/text/cea/CeaUtil.java

@@ -25,6 +25,10 @@
  */
 public final class CeaUtil {
 
+  public static final int MODE_H262 = 1;
+  public static final int MODE_H264 = 2;
+  public static final int MODE_H265 = MODE_H264;
+
   private static final String TAG = "CeaUtil";
 
   private static final int PAYLOAD_TYPE_CC = 4;
@@ -33,6 +37,26 @@ public final class CeaUtil {
   private static final int USER_ID = 0x47413934; // "GA94"
   private static final int USER_DATA_TYPE_CODE = 0x3;
 
+  public static void consume(long presentationTimeUs, ParsableByteArray buffer,
+      TrackOutput[] outputs, int mode) {
+    switch (mode) {
+      case MODE_H262:
+        consumeUserData(presentationTimeUs, buffer, outputs);
+        break;
+      case MODE_H264:
+        consumeSei(presentationTimeUs, buffer, outputs);
+        break;
+      default:
+        Log.w(TAG, "Unknown mode: " + mode);
+        break;
+    }
+  }
+
+  public static void consume(long presentationTimeUs, ParsableByteArray buffer,
+      TrackOutput[] outputs) {
+    consume(presentationTimeUs, buffer, outputs, MODE_H264);
+  }
+
   /**
    * Consumes the unescaped content of an SEI NAL unit, writing the content of any CEA-608 messages
    * as samples to all of the provided outputs.
@@ -41,7 +65,7 @@ public final class CeaUtil {
    * @param seiBuffer The unescaped SEI NAL unit data, excluding the NAL unit start code and type.
    * @param outputs The outputs to which any samples should be written.
    */
-  public static void consume(long presentationTimeUs, ParsableByteArray seiBuffer,
+  private static void consumeSei(long presentationTimeUs, ParsableByteArray seiBuffer,
       TrackOutput[] outputs) {
     while (seiBuffer.bytesLeft() > 1 /* last byte will be rbsp_trailing_bits */) {
       int payloadType = readNon255TerminatedValue(seiBuffer);
@@ -55,19 +79,8 @@ public static void consume(long presentationTimeUs, ParsableByteArray seiBuffer,
         // Ignore country_code (1) + provider_code (2) + user_identifier (4)
         // + user_data_type_code (1).
         seiBuffer.skipBytes(8);
-        // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1).
-        int ccCount = seiBuffer.readUnsignedByte() & 0x1F;
-        // Ignore em_data (1)
-        seiBuffer.skipBytes(1);
-        // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2)
-        // + cc_data_1 (8) + cc_data_2 (8).
-        int sampleLength = ccCount * 3;
-        int sampleStartPosition = seiBuffer.getPosition();
-        for (TrackOutput output : outputs) {
-          seiBuffer.setPosition(sampleStartPosition);
-          output.sampleData(seiBuffer, sampleLength);
-          output.sampleMetadata(presentationTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null);
-        }
+        // Consume closed-captions data.
+        int ccCount = consumeCea608Data(presentationTimeUs, seiBuffer, outputs);
         // Ignore trailing information in SEI, if any.
         seiBuffer.skipBytes(payloadSize - (10 + ccCount * 3));
       } else {
@@ -76,6 +89,32 @@ public static void consume(long presentationTimeUs, ParsableByteArray seiBuffer,
     }
   }
 
+  private static void consumeUserData(long presentationTimeUs, ParsableByteArray userDataBuffer,
+      TrackOutput[] outputs) {
+    if (isUserDataCea608(userDataBuffer)) {
+      userDataBuffer.skipBytes(5);
+      consumeCea608Data(presentationTimeUs, userDataBuffer, outputs);
+    }
+  }
+
+  private static int consumeCea608Data(long presentationTimeUs, ParsableByteArray buffer,
+      TrackOutput[] outputs) {
+    // Ignore first three bits: reserved (1) + process_cc_data_flag (1) + zero_bit (1).
+    int ccCount = buffer.readUnsignedByte() & 0x1F;
+    // Ignore em_data (1)
+    buffer.skipBytes(1);
+    // Each data packet consists of 24 bits: marker bits (5) + cc_valid (1) + cc_type (2)
+    // + cc_data_1 (8) + cc_data_2 (8).
+    int sampleLength = ccCount * 3;
+    int sampleStartPosition = buffer.getPosition();
+    for (TrackOutput output : outputs) {
+      buffer.setPosition(sampleStartPosition);
+      output.sampleData(buffer, sampleLength);
+      output.sampleMetadata(presentationTimeUs, C.BUFFER_FLAG_KEY_FRAME, sampleLength, 0, null);
+    }
+    return ccCount;
+  }
+
   /**
    * Reads a value from the provided buffer consisting of zero or more 0xFF bytes followed by a
    * terminating byte not equal to 0xFF. The returned value is ((0xFF * N) + T), where N is the
@@ -122,6 +161,18 @@ private static boolean isSeiMessageCea608(int payloadType, int payloadLength,
         && userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE;
   }
 
+  private static boolean isUserDataCea608(ParsableByteArray userData) {
+    if (userData.bytesLeft() < 5) {
+      // Need at least 5 bytes: USER_ID + USER_DATA_TYPE_CODE
+      return false;
+    }
+    int startPosition = userData.getPosition();
+    int userIdentifier = userData.readInt();
+    int userDataTypeCode = userData.readUnsignedByte();
+    userData.setPosition(startPosition);
+    return userIdentifier == USER_ID && userDataTypeCode == USER_DATA_TYPE_CODE;
+  }
+
   private CeaUtil() {}
 
 }