Update ID3 cues to span until playlist end, or next cue with same tag…

… type (value.key) on cue append

Resolves #3879

api-extractor/report/hls.js.api.md

@@ -813,6 +813,8 @@ export interface FragParsingInitSegmentData {
 //
 // @public (undocumented)
 export interface FragParsingMetadataData {
+    // (undocumented)
+    details: LevelDetails;
     // (undocumented)
     frag: Fragment;
     // (undocumented)
@@ -825,6 +827,8 @@ export interface FragParsingMetadataData {
 //
 // @public (undocumented)
 export interface FragParsingUserdataData {
+    // (undocumented)
+    details: LevelDetails;
     // (undocumented)
     frag: Fragment;
     // (undocumented)

docs/API.md

@@ -1483,9 +1483,9 @@ Full list of Events is available below:
 - `Hls.Events.FRAG_PARSING_INIT_SEGMENT` - fired when Init Segment has been extracted from fragment
   - data: { id: demuxer id, frag : fragment object, moov : moov MP4 box, codecs : codecs found while parsing fragment }
 - `Hls.Events.FRAG_PARSING_USERDATA` - fired when parsing sei text is completed
-  - data: { id : demuxer id, frag: fragment object, samples : [ sei samples pes ] }
+  - data: { id : demuxer id, frag: fragment object, samples : [ sei samples pes ], details: `levelDetails` object (please see [below](#leveldetails) for more information) }
 - `Hls.Events.FRAG_PARSING_METADATA` - fired when parsing id3 is completed
-  - data: { id: demuxer id, frag : fragment object, samples : [ id3 pes - pts and dts timestamp are relative, values are in seconds] }
+  - data: { id: demuxer id, frag : fragment object, samples : [ id3 pes - pts and dts timestamp are relative, values are in seconds], details: `levelDetails` object (please see [below](#leveldetails) for more information) }
 - `Hls.Events.FRAG_PARSING_DATA` - [deprecated]
 - `Hls.Events.FRAG_PARSED` - fired when fragment parsing is completed
   - data: { frag : fragment object, partIndex }

src/controller/audio-stream-controller.ts

@@ -680,12 +680,16 @@ class AudioStreamController
       this.resetLiveStartWhenNotLoaded(chunkMeta.level);
       return;
     }
-    const { frag, part } = context;
+    const {
+      frag,
+      part,
+      level: { details },
+    } = context;
     const { audio, text, id3, initSegment } = remuxResult;
 
     // Check if the current fragment has been aborted. We check this by first seeing if we're still playing the current level.
     // If we are, subsequently check if the currently loading fragment (fragCurrent) has changed.
-    if (this.fragContextChanged(frag)) {
+    if (this.fragContextChanged(frag) || !details) {
       return;
     }
 
@@ -726,8 +730,9 @@ class AudioStreamController
     if (id3?.samples?.length) {
       const emittedID3: FragParsingMetadataData = Object.assign(
         {
-          frag,
           id,
+          frag,
+          details,
         },
         id3
       );
@@ -736,8 +741,9 @@ class AudioStreamController
     if (text) {
       const emittedText: FragParsingUserdataData = Object.assign(
         {
-          frag,
           id,
+          frag,
+          details,
         },
         text
       );

src/controller/id3-track-controller.ts

@@ -92,26 +92,29 @@ class ID3TrackController implements ComponentAPI {
     if (!this.media) {
       return;
     }
-    const fragment = data.frag;
-    const samples = data.samples;
+    const { frag: fragment, samples, details } = data;
 
     // create track dynamically
     if (!this.id3Track) {
       this.id3Track = this.getID3Track(this.media.textTracks) as TextTrack;
       this.id3Track.mode = 'hidden';
     }
 
+    // VTTCue end time must be finite, so use playlist edge or fragment end until next fragment with same frame type is found
+    const maxCueTime = details.edge || fragment.end;
+
     // Attempt to recreate Safari functionality by creating
     // WebKitDataCue objects when available and store the decoded
     // ID3 data in the value property of the cue
     const Cue = (self.WebKitDataCue || self.VTTCue || self.TextTrackCue) as any;
+    let updateCueRanges = false;
+    const frameTypesAdded: Record<string, number | null> = {};
 
     for (let i = 0; i < samples.length; i++) {
       const frames = ID3.getID3Frames(samples[i].data);
       if (frames) {
         const startTime = samples[i].pts;
-        let endTime: number =
-          i < samples.length - 1 ? samples[i + 1].pts : fragment.end;
+        let endTime: number = maxCueTime;
 
         const timeDiff = endTime - startTime;
         if (timeDiff <= 0) {
@@ -125,7 +128,30 @@ class ID3TrackController implements ComponentAPI {
             const cue = new Cue(startTime, endTime, '');
             cue.value = frame;
             this.id3Track.addCue(cue);
+            frameTypesAdded[frame.key] = null;
+            updateCueRanges = true;
+          }
+        }
+      }
+    }
+    if (updateCueRanges) {
+      this.updateId3CueEnds(frameTypesAdded);
+    }
+  }
+
+  updateId3CueEnds(frameTypesAdded: Record<string, number | null>) {
+    // Update endTime of previous cue with same IDR frame.type (Ex: TXXX cue spans to next TXXX)
+    const cues = this.id3Track?.cues;
+    if (cues) {
+      for (let i = cues.length; i--; ) {
+        const cue = cues[i] as any;
+        const frameType = cue.value?.key;
+        if (frameType && frameType in frameTypesAdded) {
+          const startTime = frameTypesAdded[frameType];
+          if (startTime && cue.endTime !== startTime) {
+            cue.endTime = startTime;
           }
+          frameTypesAdded[frameType] = cue.startTime;
         }
       }
     }

src/controller/stream-controller.ts

@@ -1048,6 +1048,7 @@ export default class StreamController
     }
     const { frag, part, level } = context;
     const { video, text, id3, initSegment } = remuxResult;
+    const { details } = level;
     // The audio-stream-controller handles audio buffering if Hls.js is playing an alternate audio track
     const audio = this.altAudio ? undefined : remuxResult.audio;
 
@@ -1080,7 +1081,7 @@ export default class StreamController
 
     // Avoid buffering if backtracking this fragment
     if (video && remuxResult.independent !== false) {
-      if (level.details) {
+      if (details) {
         const { startPTS, endPTS, startDTS, endDTS } = video;
         if (part) {
           part.elementaryStreams[video.type] = {
@@ -1145,18 +1146,20 @@ export default class StreamController
       this.bufferFragmentData(audio, frag, part, chunkMeta);
     }
 
-    if (id3?.samples?.length) {
+    if (details && id3?.samples?.length) {
       const emittedID3: FragParsingMetadataData = {
-        frag,
         id,
+        frag,
+        details,
         samples: id3.samples,
       };
       hls.trigger(Events.FRAG_PARSING_METADATA, emittedID3);
     }
-    if (text) {
+    if (details && text) {
       const emittedText: FragParsingUserdataData = {
-        frag,
         id,
+        frag,
+        details,
         samples: text.samples,
       };
       hls.trigger(Events.FRAG_PARSING_USERDATA, emittedText);

src/types/events.ts

@@ -305,12 +305,14 @@ export interface FragParsingInitSegmentData {}
 export interface FragParsingUserdataData {
   id: string;
   frag: Fragment;
+  details: LevelDetails;
   samples: UserdataSample[];
 }
 
 export interface FragParsingMetadataData {
   id: string;
   frag: Fragment;
+  details: LevelDetails;
   samples: MetadataSample[];
 }