Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

muxer: fix invalid fMP4 BaseTime in case of negative DTS #111

Merged
merged 1 commit into from
Nov 22, 2023
Merged

muxer: fix invalid fMP4 BaseTime in case of negative DTS #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions muxer_part.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func (p *muxerPart) finalize(nextDTS time.Duration) error {
return nil
}

func (p *muxerPart) writeVideo(sample *augmentedVideoSample) {
func (p *muxerPart) writeVideo(sample *augmentedSample) {
if !p.videoStartDTSFilled {
p.videoStartDTSFilled = true
p.videoStartDTS = sample.dts
Expand All @@ -115,7 +115,7 @@ func (p *muxerPart) writeVideo(sample *augmentedVideoSample) {
p.videoSamples = append(p.videoSamples, &sample.PartSample)
}

func (p *muxerPart) writeAudio(sample *augmentedAudioSample) {
func (p *muxerPart) writeAudio(sample *augmentedSample) {
if !p.audioStartDTSFilled {
p.audioStartDTSFilled = true
p.audioStartDTS = sample.dts
Expand Down
4 changes: 2 additions & 2 deletions muxer_segment_fmp4.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ func (s *muxerSegmentFMP4) finalize(nextDTS time.Duration) error {
}

func (s *muxerSegmentFMP4) writeVideo(
sample *augmentedVideoSample,
sample *augmentedSample,
nextDTS time.Duration,
adjustedPartDuration time.Duration,
) error {
Expand Down Expand Up @@ -168,7 +168,7 @@ func (s *muxerSegmentFMP4) writeVideo(
}

func (s *muxerSegmentFMP4) writeAudio(
sample *augmentedAudioSample,
sample *augmentedSample,
nextAudioSampleDTS time.Duration,
adjustedPartDuration time.Duration,
) error {
Expand Down
167 changes: 92 additions & 75 deletions muxer_segmenter_fmp4.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,10 @@
"github.com/bluenviron/gohlslib/pkg/storage"
)

const (
fmp4StartDTS = 10 * time.Second
)

func fmp4TimeScale(c codecs.Codec) uint32 {
switch codec := c.(type) {
case *codecs.MPEG4Audio:
Expand Down Expand Up @@ -77,13 +81,7 @@
return nil
}

type augmentedVideoSample struct {
fmp4.PartSample
dts time.Duration
ntp time.Time
}

type augmentedAudioSample struct {
type augmentedSample struct {
fmp4.PartSample
dts time.Duration
ntp time.Time
Expand All @@ -104,11 +102,12 @@
audioTimeScale uint32
videoFirstRandomAccessReceived bool
videoDTSExtractor dtsExtractor
startDTS time.Duration
currentSegment *muxerSegmentFMP4
nextSegmentID uint64
nextPartID uint64
nextVideoSample *augmentedVideoSample
nextAudioSample *augmentedAudioSample
nextVideoSample *augmentedSample
nextAudioSample *augmentedSample
firstSegmentFinalized bool
sampleDurations map[time.Duration]struct{}
adjustedPartDuration time.Duration
Expand Down Expand Up @@ -219,16 +218,14 @@
return err
}

sample := &augmentedVideoSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
})

Check warning on line 228 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L224-L228 

Added lines #L224 - L228 were not covered by tests
}

func (m *muxerSegmenterFMP4) writeVP9(
Expand All @@ -247,19 +244,17 @@
m.videoFirstRandomAccessReceived = true
}

sample := &augmentedVideoSample{
PartSample: fmp4.PartSample{
IsNonSyncSample: !randomAccess,
Payload: frame,
},
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: fmp4.PartSample{
IsNonSyncSample: !randomAccess,
Payload: frame,
},
dts: dts,
ntp: ntp,
})

Check warning on line 257 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L250-L257 

Added lines #L250 - L257 were not covered by tests
}

func (m *muxerSegmenterFMP4) writeH26x(
Expand Down Expand Up @@ -295,23 +290,74 @@
return err
}

sample := &augmentedVideoSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
}

return m.writeVideo(
randomAccess,
forceSwitch,
sample)
&augmentedSample{
PartSample: *ps,
dts: dts,
ntp: ntp,
})
}

func (m *muxerSegmenterFMP4) writeOpus(ntp time.Time, pts time.Duration, packets [][]byte) error {
for _, packet := range packets {
err := m.writeAudio(&augmentedSample{
PartSample: fmp4.PartSample{
Payload: packet,
},
dts: pts,
ntp: ntp,
})
if err != nil {
return err
}

Check warning on line 314 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L303-L314 

Added lines #L303 - L314 were not covered by tests

duration := opus.PacketDuration(packet)
ntp = ntp.Add(duration)
pts += duration

Check warning on line 318 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L316-L318 

Added lines #L316 - L318 were not covered by tests
}

return nil

Check warning on line 321 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L321 

Added line #L321 was not covered by tests
}

func (m *muxerSegmenterFMP4) writeMPEG4Audio(ntp time.Time, pts time.Duration, aus [][]byte) error {
sampleRate := time.Duration(m.audioTrack.Codec.(*codecs.MPEG4Audio).Config.SampleRate)

for i, au := range aus {
auNTP := ntp.Add(time.Duration(i) * mpeg4audio.SamplesPerAccessUnit *
time.Second / sampleRate)
auPTS := pts + time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/sampleRate

err := m.writeAudio(&augmentedSample{
PartSample: fmp4.PartSample{
Payload: au,
},
dts: auPTS,
ntp: auNTP,
})
if err != nil {
return err
}

Check warning on line 342 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L341-L342 

Added lines #L341 - L342 were not covered by tests
}

return nil
}

func (m *muxerSegmenterFMP4) writeVideo(
randomAccess bool,
forceSwitch bool,
sample *augmentedVideoSample,
sample *augmentedSample,
) error {
// add a starting DTS to avoid a negative BaseTime
sample.dts += fmp4StartDTS

// BaseTime is still negative, this is not supported by fMP4. Reject the sample silently.
if (sample.dts - m.startDTS) < 0 {
return nil
}

Check warning on line 359 in muxer_segmenter_fmp4.go

View check run for this annotation

Codecov / codecov/patch

muxer_segmenter_fmp4.go#L358-L359 

Added lines #L358 - L359 were not covered by tests

// put samples into a queue in order to
// - compute sample duration
// - check if next sample is IDR
Expand All @@ -321,8 +367,10 @@
}
sample.Duration = uint32(durationGoToMp4(m.nextVideoSample.dts-sample.dts, 90000))

// create first segment
if m.currentSegment == nil {
// create first segment
m.startDTS = sample.dts

var err error
m.currentSegment, err = newMuxerSegmentFMP4(
m.lowLatency,
Expand Down Expand Up @@ -399,55 +447,22 @@
return nil
}

func (m *muxerSegmenterFMP4) writeOpus(ntp time.Time, pts time.Duration, packets [][]byte) error {
for _, packet := range packets {
err := m.writeAudio(ntp, pts, packet)
if err != nil {
return err
}

duration := opus.PacketDuration(packet)
ntp = ntp.Add(duration)
pts += duration
}

return nil
}
func (m *muxerSegmenterFMP4) writeAudio(sample *augmentedSample) error {
// add a starting DTS to avoid a negative BaseTime
sample.dts += fmp4StartDTS

func (m *muxerSegmenterFMP4) writeMPEG4Audio(ntp time.Time, pts time.Duration, aus [][]byte) error {
sampleRate := time.Duration(m.audioTrack.Codec.(*codecs.MPEG4Audio).Config.SampleRate)

for i, au := range aus {
auNTP := ntp.Add(time.Duration(i) * mpeg4audio.SamplesPerAccessUnit *
time.Second / sampleRate)
auPTS := pts + time.Duration(i)*mpeg4audio.SamplesPerAccessUnit*
time.Second/sampleRate

err := m.writeAudio(auNTP, auPTS, au)
if err != nil {
return err
}
// BaseTime is still negative, this is not supported by fMP4. Reject the sample silently.
if (sample.dts - m.startDTS) < 0 {
return nil
}

return nil
}

func (m *muxerSegmenterFMP4) writeAudio(ntp time.Time, dts time.Duration, au []byte) error {
if m.videoTrack != nil {
// wait for the video track
if !m.videoFirstRandomAccessReceived {
return nil
}
}

sample := &augmentedAudioSample{
PartSample: fmp4.PartSample{
Payload: au,
},
dts: dts,
ntp: ntp,
}

// put samples into a queue in order to compute the sample duration
sample, m.nextAudioSample = m.nextAudioSample, sample
if sample == nil {
Expand All @@ -456,8 +471,10 @@
sample.Duration = uint32(durationGoToMp4(m.nextAudioSample.dts-sample.dts, m.audioTimeScale))

if m.videoTrack == nil {
// create first segment
if m.currentSegment == nil {
// create first segment
m.startDTS = sample.dts

var err error
m.currentSegment, err = newMuxerSegmentFMP4(
m.lowLatency,
Expand Down
Loading
Loading