Skip to content

Commit

Permalink
Flush MSE rendering pipeline when crossing video buffer holes with audio
Browse files Browse the repository at this point in the history 
Resolves #5631
  • Loading branch information
@robwalch
robwalch committed Jan 24, 2025
1 parent ab22c5b commit 5636c46
Show file tree
Hide file tree
Showing 16 changed files with 427 additions and 259 deletions.
54 changes: 13 additions & 41 deletions api-extractor/report/hls.js.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,8 +551,6 @@ export class BaseStreamController extends TaskLoop implements NetworkComponentAP
// (undocumented)
protected transmuxer: TransmuxerInterface | null;
// (undocumented)
protected triggerEnded(): void;
// (undocumented)
protected unregisterListeners(): void;
// (undocumented)
protected waitForCdnTuneIn(details: LevelDetails): boolean | 0;
Expand Down Expand Up @@ -650,47 +648,15 @@ export interface BufferCodecsData {
export class BufferController extends Logger implements ComponentAPI {
constructor(hls: Hls, fragmentTracker: FragmentTracker);
// (undocumented)
protected appendChangeType(type: SourceBufferName, container: string, codec: string): void;
// (undocumented)
get bufferedToEnd(): boolean;
// (undocumented)
protected checkPendingTracks(): void;
// (undocumented)
destroy(): void;
// (undocumented)
hasSourceTypes(): boolean;
// (undocumented)
media: HTMLMediaElement | null;
// (undocumented)
mediaSource: MediaSource | null;
// (undocumented)
protected onBufferAppending(event: Events.BUFFER_APPENDING, eventData: BufferAppendingData): void;
// (undocumented)
protected onBufferCodecs(event: Events.BUFFER_CODECS, data: BufferCodecsData): void;
// (undocumented)
protected onBufferEos(event: Events.BUFFER_EOS, data: BufferEOSData): void;
// (undocumented)
protected onBufferFlushing(event: Events.BUFFER_FLUSHING, data: BufferFlushingData): void;
// (undocumented)
protected onBufferReset(): void;
// (undocumented)
protected onFragParsed(event: Events.FRAG_PARSED, data: FragParsedData): void;
// (undocumented)
protected onLevelUpdated(event: Events.LEVEL_UPDATED, { details }: LevelUpdatedData): void;
// (undocumented)
protected onManifestParsed(event: Events.MANIFEST_PARSED, data: ManifestParsedData): void;
// (undocumented)
protected onMediaAttaching(event: Events.MEDIA_ATTACHING, data: MediaAttachingData): void;
// (undocumented)
protected onMediaDetaching(event: Events.MEDIA_DETACHING, data: MediaDetachingData): void;
// (undocumented)
protected registerListeners(): void;
// (undocumented)
get sourceBufferTracks(): BaseTrackSet;
// (undocumented)
transferMedia(): AttachMediaSourceData | null;
// (undocumented)
protected unregisterListeners(): void;
}

// Warning: (ae-missing-release-tag) "BufferControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
Expand Down Expand Up @@ -764,6 +730,7 @@ export type BufferInfo = {
end: number;
nextStart?: number;
buffered?: BufferTimeRange[];
bufferedIndex: number;
};

// Warning: (ae-missing-release-tag) "BufferTimeRange" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
Expand Down Expand Up @@ -1958,6 +1925,17 @@ export interface FragParsingUserdataData {
samples: UserdataSample[];
}

// Warning: (ae-missing-release-tag) "GapControllerConfig" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
export type GapControllerConfig = {
detectStallWithCurrentTimeMs: number;
highBufferWatchdogPeriod: number;
nudgeOffset: number;
nudgeMaxRetry: number;
nudgeOnVideoHole: boolean;
};

// Warning: (ae-missing-release-tag) "HdcpLevel" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
// @public (undocumented)
Expand Down Expand Up @@ -2219,8 +2197,7 @@ export type HlsConfig = {
progressive: boolean;
lowLatencyMode: boolean;
primarySessionId?: string;
detectStallWithCurrentTimeMs: number;
} & ABRControllerConfig & BufferControllerConfig & CapLevelControllerConfig & EMEControllerConfig & FPSControllerConfig & LevelControllerConfig & MP4RemuxerConfig & StreamControllerConfig & SelectionPreferences & LatencyControllerConfig & MetadataControllerConfig & TimelineControllerConfig & TSDemuxerConfig & HlsLoadPolicies & FragmentLoaderConfig & PlaylistLoaderConfig;
} & ABRControllerConfig & BufferControllerConfig & CapLevelControllerConfig & EMEControllerConfig & FPSControllerConfig & GapControllerConfig & LevelControllerConfig & MP4RemuxerConfig & StreamControllerConfig & SelectionPreferences & LatencyControllerConfig & MetadataControllerConfig & TimelineControllerConfig & TSDemuxerConfig & HlsLoadPolicies & FragmentLoaderConfig & PlaylistLoaderConfig;

// Warning: (ae-missing-release-tag) "HlsEventEmitter" is part of the package's API, but it is missing a release tag (@alpha, @beta, @public, or @internal)
//
Expand Down Expand Up @@ -4416,8 +4393,6 @@ export class StreamController extends BaseStreamController implements NetworkCom
// (undocumented)
swapAudioCodec(): void;
// (undocumented)
protected triggerEnded(): void;
// (undocumented)
protected unregisterListeners(): void;
}

Expand All @@ -4432,9 +4407,6 @@ export type StreamControllerConfig = {
maxBufferLength: number;
maxBufferSize: number;
maxBufferHole: number;
highBufferWatchdogPeriod: number;
nudgeOffset: number;
nudgeMaxRetry: number;
maxFragLookUpTolerance: number;
maxMaxBufferLength: number;
startFragPrefetch: boolean;
Expand Down
24 changes: 15 additions & 9 deletions src/config.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,15 +214,20 @@ export type StreamControllerConfig = {
maxBufferLength: number;
maxBufferSize: number;
maxBufferHole: number;
highBufferWatchdogPeriod: number;
nudgeOffset: number;
nudgeMaxRetry: number;
maxFragLookUpTolerance: number;
maxMaxBufferLength: number;
startFragPrefetch: boolean;
testBandwidth: boolean;
};

export type GapControllerConfig = {
detectStallWithCurrentTimeMs: number;
highBufferWatchdogPeriod: number;
nudgeOffset: number;
nudgeMaxRetry: number;
nudgeOnVideoHole: boolean;
};

export type SelectionPreferences = {
videoPreference?: VideoSelectionOption;
audioPreference?: AudioSelectionOption;
Expand Down Expand Up @@ -319,12 +324,12 @@ export type HlsConfig = {
progressive: boolean;
lowLatencyMode: boolean;
primarySessionId?: string;
detectStallWithCurrentTimeMs: number;
} & ABRControllerConfig &
BufferControllerConfig &
CapLevelControllerConfig &
EMEControllerConfig &
FPSControllerConfig &
GapControllerConfig &
LevelControllerConfig &
MP4RemuxerConfig &
StreamControllerConfig &
Expand Down Expand Up @@ -365,11 +370,13 @@ export const hlsDefaultConfig: HlsConfig = {
backBufferLength: Infinity, // used by buffer-controller
frontBufferFlushThreshold: Infinity,
maxBufferSize: 60 * 1000 * 1000, // used by stream-controller
maxBufferHole: 0.1, // used by stream-controller
highBufferWatchdogPeriod: 2, // used by stream-controller
nudgeOffset: 0.1, // used by stream-controller
nudgeMaxRetry: 3, // used by stream-controller
maxFragLookUpTolerance: 0.25, // used by stream-controller
maxBufferHole: 0.1, // used by stream-controller and gap-controller
detectStallWithCurrentTimeMs: 1250, // used by gap-controller
highBufferWatchdogPeriod: 2, // used by gap-controller
nudgeOffset: 0.1, // used by gap-controller
nudgeMaxRetry: 3, // used by gap-controller
nudgeOnVideoHole: true, // used by gap-controller
liveSyncDurationCount: 3, // used by latency-controller
liveSyncOnStallIncrease: 1, // used by latency-controller
liveMaxLatencyDurationCount: Infinity, // used by latency-controller
Expand Down Expand Up @@ -428,7 +435,6 @@ export const hlsDefaultConfig: HlsConfig = {
progressive: false,
lowLatencyMode: true,
cmcd: undefined,
detectStallWithCurrentTimeMs: 1250,
enableDateRangeMetadataCues: true,
enableEmsgMetadataCues: true,
enableEmsgKLVMetadata: false,
Expand Down
5 changes: 0 additions & 5 deletions src/controller/base-stream-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -391,13 +391,8 @@ export default class BaseStreamController
// reset startPosition and lastCurrentTime to restart playback @ stream beginning
this.log(`setting startPosition to 0 because media ended`);
this.startPosition = this.lastCurrentTime = 0;
this.triggerEnded();
};

protected triggerEnded() {
/* overridden in stream-controller */
}

protected onManifestLoaded(
event: Events.MANIFEST_LOADED,
data: ManifestLoadedData,
Expand Down
41 changes: 19 additions & 22 deletions src/controller/buffer-controller.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ export default class BufferController extends Logger implements ComponentAPI {
private bufferCodecEventsTotal: number = 0;

// A reference to the attached media element
public media: HTMLMediaElement | null = null;
private media: HTMLMediaElement | null = null;

// A reference to the active media source
public mediaSource: MediaSource | null = null;
private mediaSource: MediaSource | null = null;

// Last MP3 audio chunk appended
private lastMpegAudioChunk: ChunkMetadata | null = null;
Expand Down Expand Up @@ -152,7 +152,7 @@ export default class BufferController extends Logger implements ComponentAPI {
this._onStartStreaming = this._onEndStreaming = null;
}

protected registerListeners() {
private registerListeners() {
const { hls } = this;
hls.on(Events.MEDIA_ATTACHING, this.onMediaAttaching, this);
hls.on(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
Expand All @@ -169,7 +169,7 @@ export default class BufferController extends Logger implements ComponentAPI {
hls.on(Events.ERROR, this.onError, this);
}

protected unregisterListeners() {
private unregisterListeners() {
const { hls } = this;
hls.off(Events.MEDIA_ATTACHING, this.onMediaAttaching, this);
hls.off(Events.MEDIA_DETACHING, this.onMediaDetaching, this);
Expand Down Expand Up @@ -247,7 +247,7 @@ export default class BufferController extends Logger implements ComponentAPI {
this.details = null;
}

protected onManifestParsed(
private onManifestParsed(
event: Events.MANIFEST_PARSED,
data: ManifestParsedData,
) {
Expand All @@ -270,7 +270,7 @@ export default class BufferController extends Logger implements ComponentAPI {
}
}

protected onMediaAttaching(
private onMediaAttaching(
event: Events.MEDIA_ATTACHING,
data: MediaAttachingData,
) {
Expand Down Expand Up @@ -440,14 +440,15 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}
this.hls.pauseBuffering();
};

private _onStartStreaming = (event) => {
if (!this.hls) {
return;
}
this.hls.resumeBuffering();
};

protected onMediaDetaching(
private onMediaDetaching(
event: Events.MEDIA_DETACHING,
data: MediaDetachingData,
) {
Expand Down Expand Up @@ -540,7 +541,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
this.hls.trigger(Events.MEDIA_DETACHED, data);
}

protected onBufferReset() {
private onBufferReset() {
this.sourceBuffers.forEach(([type]) => {
if (type) {
this.resetBuffer(type);
Expand Down Expand Up @@ -580,10 +581,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
this.operationQueue = new BufferOperationQueue(this.tracks);
}

protected onBufferCodecs(
event: Events.BUFFER_CODECS,
data: BufferCodecsData,
) {
private onBufferCodecs(event: Events.BUFFER_CODECS, data: BufferCodecsData) {
const tracks = this.tracks;
const trackNames = Object.keys(data);
this.log(
Expand Down Expand Up @@ -614,7 +612,6 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
const sbTrack = transferredTrack?.buffer ? transferredTrack : track;
const sbCodec = sbTrack?.pendingCodec || sbTrack?.codec;
const trackLevelCodec = sbTrack?.levelCodec;
const forceChangeType = !sbTrack || !!this.hls.config.assetPlayerId;
if (!track) {
track = tracks[trackName] = {
buffer: undefined,
Expand All @@ -637,7 +634,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
);
let trackCodec = pickMostCompleteCodecName(codec, levelCodec);
const nextCodec = trackCodec?.replace(VIDEO_CODEC_PROFILE_REPLACE, '$1');
if (trackCodec && (currentCodec !== nextCodec || forceChangeType)) {
if (trackCodec && currentCodecFull && currentCodec !== nextCodec) {
if (trackName.slice(0, 5) === 'audio') {
trackCodec = getCodecCompatibleName(trackCodec, this.appendSource);
}
Expand Down Expand Up @@ -676,7 +673,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}, {});
}

protected appendChangeType(
private appendChangeType(
type: SourceBufferName,
container: string,
codec: string,
Expand Down Expand Up @@ -748,7 +745,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}
}

protected onBufferAppending(
private onBufferAppending(
event: Events.BUFFER_APPENDING,
eventData: BufferAppendingData,
) {
Expand Down Expand Up @@ -956,7 +953,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
};
}

protected onBufferFlushing(
private onBufferFlushing(
event: Events.BUFFER_FLUSHING,
data: BufferFlushingData,
) {
Expand All @@ -972,7 +969,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}
}

protected onFragParsed(event: Events.FRAG_PARSED, data: FragParsedData) {
private onFragParsed(event: Events.FRAG_PARSED, data: FragParsedData) {
const { frag, part } = data;
const buffersAppendedTo: SourceBufferName[] = [];
const elementaryStreams = part
Expand Down Expand Up @@ -1017,7 +1014,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
this.trimBuffers();
}

get bufferedToEnd(): boolean {
public get bufferedToEnd(): boolean {
return (
this.sourceBufferCount > 0 &&
!this.sourceBuffers.some(
Expand All @@ -1029,7 +1026,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i

// on BUFFER_EOS mark matching sourcebuffer(s) as "ending" and "ended" and queue endOfStream after remaining operations(s)
// an undefined data.type will mark all buffers as EOS.
protected onBufferEos(event: Events.BUFFER_EOS, data: BufferEOSData) {
private onBufferEos(event: Events.BUFFER_EOS, data: BufferEOSData) {
this.sourceBuffers.forEach(([type]) => {
if (type) {
const track = this.tracks[type] as SourceBufferTrack;
Expand Down Expand Up @@ -1078,7 +1075,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
}
}

protected onLevelUpdated(
private onLevelUpdated(
event: Events.LEVEL_UPDATED,
{ details }: LevelUpdatedData,
) {
Expand Down Expand Up @@ -1321,7 +1318,7 @@ transfer tracks: ${JSON.stringify(transferredTracks, (key, value) => (key === 'i
);
}

protected checkPendingTracks() {
private checkPendingTracks() {
const { bufferCodecEventsTotal, pendingTrackCount, tracks } = this;
this.log(
`checkPendingTracks (pending: ${pendingTrackCount} codec events expected: ${bufferCodecEventsTotal}) ${JSON.stringify(tracks)}`,
Expand Down
Loading

0 comments on commit 5636c46

Please sign in to comment.