Skip to content

Commit d20c233

Browse files
icbakercopybara-github
icbaker
authored and
copybara-github
committedOct 10, 2023
Add experimental opt-in to parse DASH subtitles during extraction
This currently only applies to subtitles muxed into mp4 segments, and not standalone text files linked directly from the manifest. Issue: androidx/media#288 #minor-release PiperOrigin-RevId: 572263764
1 parent a2c37c2 commit d20c233

File tree

8 files changed

+163
-41
lines changed

8 files changed

+163
-41
lines changed
 

‎library/core/src/main/java/com/google/android/exoplayer2/source/chunk/BundledChunkExtractor.java

+65-29
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import androidx.annotation.Nullable;
2222
import com.google.android.exoplayer2.C;
2323
import com.google.android.exoplayer2.Format;
24+
import com.google.android.exoplayer2.analytics.PlayerId;
2425
import com.google.android.exoplayer2.extractor.ChunkIndex;
2526
import com.google.android.exoplayer2.extractor.DummyTrackOutput;
2627
import com.google.android.exoplayer2.extractor.Extractor;
@@ -31,11 +32,14 @@
3132
import com.google.android.exoplayer2.extractor.TrackOutput;
3233
import com.google.android.exoplayer2.extractor.mkv.MatroskaExtractor;
3334
import com.google.android.exoplayer2.extractor.mp4.FragmentedMp4Extractor;
35+
import com.google.android.exoplayer2.text.SubtitleParser;
36+
import com.google.android.exoplayer2.text.SubtitleTranscodingExtractor;
3437
import com.google.android.exoplayer2.upstream.DataReader;
3538
import com.google.android.exoplayer2.util.Assertions;
3639
import com.google.android.exoplayer2.util.MimeTypes;
3740
import com.google.android.exoplayer2.util.ParsableByteArray;
3841
import java.io.IOException;
42+
import java.util.List;
3943
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
4044

4145
/**
@@ -50,36 +54,68 @@
5054
@Deprecated
5155
public final class BundledChunkExtractor implements ExtractorOutput, ChunkExtractor {
5256

53-
/** {@link ChunkExtractor.Factory} for instances of this class. */
54-
public static final ChunkExtractor.Factory FACTORY =
55-
(primaryTrackType,
56-
format,
57-
enableEventMessageTrack,
58-
closedCaptionFormats,
59-
playerEmsgTrackOutput,
60-
playerId) -> {
61-
@Nullable String containerMimeType = format.containerMimeType;
62-
Extractor extractor;
63-
if (MimeTypes.isText(containerMimeType)) {
64-
// Text types do not need an extractor.
65-
return null;
66-
} else if (MimeTypes.isMatroska(containerMimeType)) {
67-
extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
68-
} else {
69-
int flags = 0;
70-
if (enableEventMessageTrack) {
71-
flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
72-
}
73-
extractor =
74-
new FragmentedMp4Extractor(
75-
flags,
76-
/* timestampAdjuster= */ null,
77-
/* sideloadedTrack= */ null,
78-
closedCaptionFormats,
79-
playerEmsgTrackOutput);
57+
/** {@link ChunkExtractor.Factory} for {@link BundledChunkExtractor}. */
58+
public static final class Factory implements ChunkExtractor.Factory {
59+
60+
/** Non-null if subtitles should be parsed during extraction, null otherwise. */
61+
@Nullable private SubtitleParser.Factory subtitleParserFactory;
62+
63+
/**
64+
* Sets the {@link SubtitleParser.Factory} to use for parsing subtitles during extraction, or
65+
* null to parse subtitles during decoding. The default is null (subtitles parsed after
66+
* decoding).
67+
*
68+
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
69+
* in a future release.
70+
*
71+
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
72+
* extraction.
73+
* @return This factory, for convenience.
74+
*/
75+
public Factory experimentalSetSubtitleParserFactory(
76+
@Nullable SubtitleParser.Factory subtitleParserFactory) {
77+
this.subtitleParserFactory = subtitleParserFactory;
78+
return this;
79+
}
80+
81+
@Nullable
82+
@Override
83+
public ChunkExtractor createProgressiveMediaExtractor(
84+
@C.TrackType int primaryTrackType,
85+
Format representationFormat,
86+
boolean enableEventMessageTrack,
87+
List<Format> closedCaptionFormats,
88+
@Nullable TrackOutput playerEmsgTrackOutput,
89+
PlayerId playerId) {
90+
@Nullable String containerMimeType = representationFormat.containerMimeType;
91+
Extractor extractor;
92+
if (MimeTypes.isText(containerMimeType)) {
93+
// Text types do not need an extractor.
94+
return null;
95+
} else if (MimeTypes.isMatroska(containerMimeType)) {
96+
extractor = new MatroskaExtractor(MatroskaExtractor.FLAG_DISABLE_SEEK_FOR_CUES);
97+
} else {
98+
int flags = 0;
99+
if (enableEventMessageTrack) {
100+
flags |= FragmentedMp4Extractor.FLAG_ENABLE_EMSG_TRACK;
80101
}
81-
return new BundledChunkExtractor(extractor, primaryTrackType, format);
82-
};
102+
extractor =
103+
new FragmentedMp4Extractor(
104+
flags,
105+
/* timestampAdjuster= */ null,
106+
/* sideloadedTrack= */ null,
107+
closedCaptionFormats,
108+
playerEmsgTrackOutput);
109+
}
110+
if (subtitleParserFactory != null) {
111+
extractor = new SubtitleTranscodingExtractor(extractor, subtitleParserFactory);
112+
}
113+
return new BundledChunkExtractor(extractor, primaryTrackType, representationFormat);
114+
}
115+
}
116+
117+
/** {@link Factory} for {@link BundledChunkExtractor}. */
118+
public static final Factory FACTORY = new Factory();
83119

84120
private static final PositionHolder POSITION_HOLDER = new PositionHolder();
85121

‎library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaPeriod.java

+26-4
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import com.google.android.exoplayer2.source.dash.manifest.EventStream;
5050
import com.google.android.exoplayer2.source.dash.manifest.Period;
5151
import com.google.android.exoplayer2.source.dash.manifest.Representation;
52+
import com.google.android.exoplayer2.text.SubtitleParser;
5253
import com.google.android.exoplayer2.trackselection.ExoTrackSelection;
5354
import com.google.android.exoplayer2.upstream.Allocator;
5455
import com.google.android.exoplayer2.upstream.CmcdConfiguration;
@@ -138,7 +139,8 @@ public DashMediaPeriod(
138139
Allocator allocator,
139140
CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory,
140141
PlayerEmsgCallback playerEmsgCallback,
141-
PlayerId playerId) {
142+
PlayerId playerId,
143+
@Nullable SubtitleParser.Factory subtitleParserFactory) {
142144
this.id = id;
143145
this.manifest = manifest;
144146
this.baseUrlExclusionList = baseUrlExclusionList;
@@ -164,7 +166,8 @@ public DashMediaPeriod(
164166
Period period = manifest.getPeriod(periodIndex);
165167
eventStreams = period.eventStreams;
166168
Pair<TrackGroupArray, TrackGroupInfo[]> result =
167-
buildTrackGroups(drmSessionManager, period.adaptationSets, eventStreams);
169+
buildTrackGroups(
170+
drmSessionManager, subtitleParserFactory, period.adaptationSets, eventStreams);
168171
trackGroups = result.first;
169172
trackGroupInfos = result.second;
170173
}
@@ -509,6 +512,7 @@ private int getPrimaryStreamIndex(int embeddedStreamIndex, int[] streamIndexToTr
509512

510513
private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
511514
DrmSessionManager drmSessionManager,
515+
@Nullable SubtitleParser.Factory subtitleParserFactory,
512516
List<AdaptationSet> adaptationSets,
513517
List<EventStream> eventStreams) {
514518
int[][] groupedAdaptationSetIndices = getGroupedAdaptationSetIndices(adaptationSets);
@@ -531,6 +535,7 @@ private static Pair<TrackGroupArray, TrackGroupInfo[]> buildTrackGroups(
531535
int trackGroupCount =
532536
buildPrimaryAndEmbeddedTrackGroupInfos(
533537
drmSessionManager,
538+
subtitleParserFactory,
534539
adaptationSets,
535540
groupedAdaptationSetIndices,
536541
primaryGroupCount,
@@ -670,6 +675,7 @@ private static int identifyEmbeddedTracks(
670675

671676
private static int buildPrimaryAndEmbeddedTrackGroupInfos(
672677
DrmSessionManager drmSessionManager,
678+
@Nullable SubtitleParser.Factory subtitleParserFactory,
673679
List<AdaptationSet> adaptationSets,
674680
int[][] groupedAdaptationSetIndices,
675681
int primaryGroupCount,
@@ -686,8 +692,24 @@ private static int buildPrimaryAndEmbeddedTrackGroupInfos(
686692
}
687693
Format[] formats = new Format[representations.size()];
688694
for (int j = 0; j < formats.length; j++) {
689-
Format format = representations.get(j).format;
690-
formats[j] = format.copyWithCryptoType(drmSessionManager.getCryptoType(format));
695+
Format originalFormat = representations.get(j).format;
696+
Format.Builder updatedFormat =
697+
originalFormat
698+
.buildUpon()
699+
.setCryptoType(drmSessionManager.getCryptoType(originalFormat));
700+
if (subtitleParserFactory != null && subtitleParserFactory.supportsFormat(originalFormat)) {
701+
updatedFormat
702+
.setSampleMimeType(MimeTypes.APPLICATION_MEDIA3_CUES)
703+
.setCueReplacementBehavior(
704+
subtitleParserFactory.getCueReplacementBehavior(originalFormat))
705+
.setCodecs(
706+
originalFormat.sampleMimeType
707+
+ (originalFormat.codecs != null ? " " + originalFormat.codecs : ""))
708+
// Reset this value to the default. All non-default timestamp adjustments are done
709+
// by SubtitleTranscodingExtractor and there are no 'subsamples' after transcoding.
710+
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE);
711+
}
712+
formats[j] = updatedFormat.build();
691713
}
692714

693715
AdaptationSet firstAdaptationSet = adaptationSets.get(adaptationSetIndices[0]);

‎library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DashMediaSource.java

+44-1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@
6060
import com.google.android.exoplayer2.source.dash.manifest.Period;
6161
import com.google.android.exoplayer2.source.dash.manifest.Representation;
6262
import com.google.android.exoplayer2.source.dash.manifest.UtcTimingElement;
63+
import com.google.android.exoplayer2.text.DefaultSubtitleParserFactory;
64+
import com.google.android.exoplayer2.text.SubtitleParser;
6365
import com.google.android.exoplayer2.upstream.Allocator;
6466
import com.google.android.exoplayer2.upstream.CmcdConfiguration;
6567
import com.google.android.exoplayer2.upstream.DataSource;
@@ -118,6 +120,7 @@ public static final class Factory implements MediaSourceFactory {
118120
private DrmSessionManagerProvider drmSessionManagerProvider;
119121
private CompositeSequenceableLoaderFactory compositeSequenceableLoaderFactory;
120122
private LoadErrorHandlingPolicy loadErrorHandlingPolicy;
123+
@Nullable private SubtitleParser.Factory subtitleParserFactory;
121124
private long fallbackTargetLiveOffsetMs;
122125
private long minLiveStartPositionUs;
123126
@Nullable private ParsingLoadable.Parser<? extends DashManifest> manifestParser;
@@ -202,6 +205,40 @@ public Factory setLoadErrorHandlingPolicy(LoadErrorHandlingPolicy loadErrorHandl
202205
return this;
203206
}
204207

208+
/**
209+
* Sets whether subtitles should be parsed as part of extraction (before the sample queue) or as
210+
* part of rendering (after the sample queue). Defaults to false (i.e. subtitles will be parsed
211+
* as part of rendering).
212+
*
213+
* <p>This method is experimental. Its default value may change, or it may be renamed or removed
214+
* in a future release.
215+
*
216+
* <p>This method may only be used with {@link DefaultDashChunkSource.Factory}.
217+
*
218+
* @param parseSubtitlesDuringExtraction Whether to parse subtitles during extraction or
219+
* rendering.
220+
* @return This factory, for convenience.
221+
*/
222+
// TODO: b/289916598 - Flip the default of this to true (probably wired up to a single method on
223+
// DefaultMediaSourceFactory via the MediaSource.Factory interface).
224+
public Factory experimentalParseSubtitlesDuringExtraction(
225+
boolean parseSubtitlesDuringExtraction) {
226+
if (parseSubtitlesDuringExtraction) {
227+
if (subtitleParserFactory == null) {
228+
this.subtitleParserFactory = new DefaultSubtitleParserFactory();
229+
}
230+
} else {
231+
this.subtitleParserFactory = null;
232+
}
233+
if (chunkSourceFactory instanceof DefaultDashChunkSource.Factory) {
234+
((DefaultDashChunkSource.Factory) chunkSourceFactory)
235+
.setSubtitleParserFactory(subtitleParserFactory);
236+
} else {
237+
throw new IllegalStateException();
238+
}
239+
return this;
240+
}
241+
205242
/**
206243
* Sets the target {@link Player#getCurrentLiveOffset() offset for live streams} that is used if
207244
* no value is defined in the {@link MediaItem} or the manifest.
@@ -321,6 +358,7 @@ public DashMediaSource createMediaSource(DashManifest manifest, MediaItem mediaI
321358
cmcdConfiguration,
322359
drmSessionManagerProvider.get(mediaItem),
323360
loadErrorHandlingPolicy,
361+
subtitleParserFactory,
324362
fallbackTargetLiveOffsetMs,
325363
minLiveStartPositionUs);
326364
}
@@ -359,6 +397,7 @@ public DashMediaSource createMediaSource(MediaItem mediaItem) {
359397
cmcdConfiguration,
360398
drmSessionManagerProvider.get(mediaItem),
361399
loadErrorHandlingPolicy,
400+
subtitleParserFactory,
362401
fallbackTargetLiveOffsetMs,
363402
minLiveStartPositionUs);
364403
}
@@ -417,6 +456,7 @@ public DashMediaSource createMediaSource(MediaItem mediaItem) {
417456
private final Runnable simulateManifestRefreshRunnable;
418457
private final PlayerEmsgCallback playerEmsgCallback;
419458
private final LoaderErrorThrower manifestLoadErrorThrower;
459+
@Nullable private final SubtitleParser.Factory subtitleParserFactory;
420460

421461
private DataSource dataSource;
422462
private Loader loader;
@@ -452,6 +492,7 @@ private DashMediaSource(
452492
@Nullable CmcdConfiguration cmcdConfiguration,
453493
DrmSessionManager drmSessionManager,
454494
LoadErrorHandlingPolicy loadErrorHandlingPolicy,
495+
@Nullable SubtitleParser.Factory subtitleParserFactory,
455496
long fallbackTargetLiveOffsetMs,
456497
long minLiveStartPositionUs) {
457498
this.mediaItem = mediaItem;
@@ -465,6 +506,7 @@ private DashMediaSource(
465506
this.cmcdConfiguration = cmcdConfiguration;
466507
this.drmSessionManager = drmSessionManager;
467508
this.loadErrorHandlingPolicy = loadErrorHandlingPolicy;
509+
this.subtitleParserFactory = subtitleParserFactory;
468510
this.fallbackTargetLiveOffsetMs = fallbackTargetLiveOffsetMs;
469511
this.minLiveStartPositionUs = minLiveStartPositionUs;
470512
this.compositeSequenceableLoaderFactory = compositeSequenceableLoaderFactory;
@@ -570,7 +612,8 @@ public MediaPeriod createPeriod(MediaPeriodId id, Allocator allocator, long star
570612
allocator,
571613
compositeSequenceableLoaderFactory,
572614
playerEmsgCallback,
573-
getPlayerId());
615+
getPlayerId(),
616+
subtitleParserFactory);
574617
periodsById.put(mediaPeriod.id, mediaPeriod);
575618
return mediaPeriod;
576619
}

‎library/dash/src/main/java/com/google/android/exoplayer2/source/dash/DefaultDashChunkSource.java

+17
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@
4848
import com.google.android.exoplayer2.source.dash.manifest.DashManifest;
4949
import com.google.android.exoplayer2.source.dash.manifest.RangedUri;
5050
import com.google.android.exoplayer2.source.dash.manifest.Representation;
51+
import com.google.android.exoplayer2.text.SubtitleParser;
5152
import com.google.android.exoplayer2.trackselection.ExoTrackSelection;
5253
import com.google.android.exoplayer2.upstream.CmcdConfiguration;
5354
import com.google.android.exoplayer2.upstream.CmcdData;
@@ -77,6 +78,7 @@
7778
@Deprecated
7879
public class DefaultDashChunkSource implements DashChunkSource {
7980

81+
/** {@link DashChunkSource.Factory} for {@link DefaultDashChunkSource} instances. */
8082
public static final class Factory implements DashChunkSource.Factory {
8183

8284
private final DataSource.Factory dataSourceFactory;
@@ -116,6 +118,21 @@ public Factory(
116118
this.maxSegmentsPerLoad = maxSegmentsPerLoad;
117119
}
118120

121+
/**
122+
* Sets the {@link SubtitleParser.Factory} to be used for parsing subtitles during extraction,
123+
* or null to parse subtitles during decoding.
124+
*
125+
* <p>This may only be used with {@link BundledChunkExtractor.Factory}.
126+
*/
127+
/* package */ Factory setSubtitleParserFactory(
128+
@Nullable SubtitleParser.Factory subtitleParserFactory) {
129+
if (chunkExtractorFactory instanceof BundledChunkExtractor.Factory) {
130+
((BundledChunkExtractor.Factory) chunkExtractorFactory)
131+
.experimentalSetSubtitleParserFactory(subtitleParserFactory);
132+
}
133+
return this;
134+
}
135+
119136
@Override
120137
public DashChunkSource createDashChunkSource(
121138
LoaderErrorThrower manifestLoaderErrorThrower,

‎library/dash/src/test/java/com/google/android/exoplayer2/source/dash/DashMediaPeriodTest.java

+2-1
Original file line numberDiff line numberDiff line change
@@ -224,7 +224,8 @@ private static DashMediaPeriod createDashMediaPeriod(DashManifest manifest, int
224224
mock(Allocator.class),
225225
mock(CompositeSequenceableLoaderFactory.class),
226226
mock(PlayerEmsgCallback.class),
227-
PlayerId.UNSET);
227+
PlayerId.UNSET,
228+
/* subtitleParserFactory= */ null);
228229
}
229230

230231
private static DashManifest parseManifest(String fileName) throws IOException {

‎library/dash/src/test/java/com/google/android/exoplayer2/source/dash/e2etest/DashPlaybackTest.java

+5-2
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@
3232
import com.google.android.exoplayer2.robolectric.PlaybackOutput;
3333
import com.google.android.exoplayer2.robolectric.ShadowMediaCodecConfig;
3434
import com.google.android.exoplayer2.robolectric.TestPlayerRunHelper;
35+
import com.google.android.exoplayer2.source.dash.DashMediaSource;
3536
import com.google.android.exoplayer2.testutil.CapturingRenderersFactory;
3637
import com.google.android.exoplayer2.testutil.DumpFileAsserts;
3738
import com.google.android.exoplayer2.testutil.FakeClock;
3839
import com.google.android.exoplayer2.trackselection.DefaultTrackSelector;
40+
import com.google.android.exoplayer2.upstream.DefaultDataSource;
3941
import org.junit.Ignore;
4042
import org.junit.Rule;
4143
import org.junit.Test;
@@ -79,14 +81,15 @@ public void ttmlStandaloneXmlFile() throws Exception {
7981

8082
// https://github.com/google/ExoPlayer/issues/7985
8183
@Test
82-
@Ignore(
83-
"Disabled until subtitles are reliably asserted in robolectric tests [internal b/174661563].")
8484
public void webvttInMp4() throws Exception {
8585
Context applicationContext = ApplicationProvider.getApplicationContext();
8686
CapturingRenderersFactory capturingRenderersFactory =
8787
new CapturingRenderersFactory(applicationContext);
8888
ExoPlayer player =
8989
new ExoPlayer.Builder(applicationContext, capturingRenderersFactory)
90+
.setMediaSourceFactory(
91+
new DashMediaSource.Factory(new DefaultDataSource.Factory(applicationContext))
92+
.experimentalParseSubtitlesDuringExtraction(true))
9093
.setClock(new FakeClock(/* isAutoAdvancing= */ true))
9194
.build();
9295
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));

0 commit comments

Comments
 (0)