Skip to content

Commit

Permalink
h264: support frame_mbs_only_flag = 0 (#68)
Browse files Browse the repository at this point in the history
  • Loading branch information
@aler9
aler9 authored Aug 29, 2023
1 parent 139216d commit 328cb87
Show file tree
Hide file tree
Showing 7 changed files with 179 additions and 31 deletions.
6 changes: 1 addition & 5 deletions pkg/codecs/h264/dts_extractor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,6 @@ func getPictureOrderCount(buf []byte, sps *SPS) (uint32, error) {
return 0, err
}

if !sps.FrameMbsOnlyFlag {
return 0, fmt.Errorf("frame_mbs_only_flag = 0 is not supported")
}

picOrderCntLsb, err := bits.ReadBits(buf, &pos, int(sps.Log2MaxPicOrderCntLsbMinus4+4))
if err != nil {
return 0, err
Expand Down Expand Up @@ -116,7 +112,7 @@ func (d *DTSExtractor) extractInner(au [][]byte, pts time.Duration) (time.Durati
return 0, false, fmt.Errorf("SPS not received yet")
}

if d.spsp.PicOrderCntType == 2 {
if d.spsp.PicOrderCntType == 2 || !d.spsp.FrameMbsOnlyFlag {
return pts, false, nil
}

Expand Down
39 changes: 39 additions & 0 deletions pkg/codecs/h264/dts_extractor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,45 @@ func TestDTSExtractor(t *testing.T) {
},
},
},
{
"mbs_only_flag = 0",
[]sample{
{
[][]byte{
{ // SPS
0x67, 0x4d, 0x40, 0x28, 0xab, 0x60, 0x3c, 0x02,
0x23, 0xef, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00,
0x10, 0x00, 0x00, 0x03, 0x03, 0x2e, 0x94, 0x00,
0x35, 0x64, 0x06, 0xb2, 0x85, 0x08, 0x0e, 0xe2,
0xc5, 0x22, 0xc0,
},
{0x68, 0xca, 0x41, 0xf2}, // PPS
{0x6, 0x0, 0x6, 0x85, 0x7e, 0x40, 0x0, 0x0, 0x10, 0x1}, // SEI
{0x65, 0x88, 0x82, 0x80, 0x1f, 0xff, 0xfb, 0xf0, 0xa2, 0x88}, // IDR
{0x6, 0x1, 0x2, 0x4, 0x24, 0x80}, // SEI
{0x41, 0x9a, 0xc, 0x1c, 0x2f, 0xe4, 0xed, 0x23, 0xb5, 0x63}, // non-IDR
},
0,
0,
},
{
[][]byte{
{0x6, 0x1, 0x2, 0x8, 0x14, 0x80}, // SEI
{0x41, 0x9a, 0x18, 0x2a, 0x1f, 0xeb, 0x2f, 0xa2, 0xb1, 0x7e}, // non-IDR
},
40 * time.Millisecond,
40 * time.Millisecond,
},
{
[][]byte{
{0x6, 0x1, 0x2, 0xc, 0x24, 0x80}, // SEI
{0x41, 0x9a, 0x1c, 0x3a, 0xf, 0xfa, 0x55, 0xc2, 0x55, 0xea}, // non-IDR
},
80 * time.Millisecond,
80 * time.Millisecond,
},
},
},
} {
t.Run(ca.name, func(t *testing.T) {
ex := NewDTSExtractor()
Expand Down
0 pkg/codecs/h264/idrpresent.go → pkg/codecs/h264/idr_present.go
View file
File renamed without changes.
0 pkg/codecs/h264/idrpresent_test.go → pkg/codecs/h264/idr_present_test.go
View file
File renamed without changes.
82 changes: 70 additions & 12 deletions pkg/codecs/h264/sps.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ type SPS struct {
ID uint32

// only for selected ProfileIdcs
ChromeFormatIdc uint32
ChromaFormatIdc uint32
SeparateColourPlaneFlag bool
BitDepthLumaMinus8 uint32
BitDepthChromaMinus8 uint32
Expand Down Expand Up @@ -473,12 +473,12 @@ func (s *SPS) Unmarshal(buf []byte) error {

switch s.ProfileIdc {
case 100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135:
s.ChromeFormatIdc, err = bits.ReadGolombUnsigned(buf, &pos)
s.ChromaFormatIdc, err = bits.ReadGolombUnsigned(buf, &pos)
if err != nil {
return err
}

if s.ChromeFormatIdc == 3 {
if s.ChromaFormatIdc == 3 {
s.SeparateColourPlaneFlag, err = bits.ReadFlag(buf, &pos)
if err != nil {
return err
Expand Down Expand Up @@ -509,7 +509,7 @@ func (s *SPS) Unmarshal(buf []byte) error {

if seqScalingMatrixPresentFlag {
var lim int
if s.ChromeFormatIdc != 3 {
if s.ChromaFormatIdc != 3 {
lim = 8
} else {
lim = 12
Expand Down Expand Up @@ -546,7 +546,7 @@ func (s *SPS) Unmarshal(buf []byte) error {
}

default:
s.ChromeFormatIdc = 0
s.ChromaFormatIdc = 0
s.SeparateColourPlaneFlag = false
s.BitDepthLumaMinus8 = 0
s.BitDepthChromaMinus8 = 0
Expand Down Expand Up @@ -697,26 +697,84 @@ func (s *SPS) Unmarshal(buf []byte) error {

// Width returns the video width.
func (s SPS) Width() int {
var subWidthC uint32
switch {
case s.ChromaFormatIdc == 1 && !s.SeparateColourPlaneFlag:
subWidthC = 2

case s.ChromaFormatIdc == 2 && !s.SeparateColourPlaneFlag:
subWidthC = 2

case s.ChromaFormatIdc == 3 && !s.SeparateColourPlaneFlag:
subWidthC = 1
}

var chromaArrayType uint32
if !s.SeparateColourPlaneFlag {
chromaArrayType = s.ChromaFormatIdc
} else {
chromaArrayType = 0
}

var cropUnitX uint32
if chromaArrayType == 0 {
cropUnitX = 0
} else {
cropUnitX = subWidthC
}

picWidthInSamplesL := ((s.PicWidthInMbsMinus1 + 1) * 16)

if s.FrameCropping != nil {
return int(((s.PicWidthInMbsMinus1 + 1) * 16) - (s.FrameCropping.LeftOffset+s.FrameCropping.RightOffset)*2)
return int(picWidthInSamplesL - cropUnitX*(s.FrameCropping.LeftOffset+s.FrameCropping.RightOffset))
}

return int((s.PicWidthInMbsMinus1 + 1) * 16)
return int(picWidthInSamplesL)
}

// Height returns the video height.
func (s SPS) Height() int {
f := uint32(0)
var subHeightC uint32
switch {
case s.ChromaFormatIdc == 1 && !s.SeparateColourPlaneFlag:
subHeightC = 2

case s.ChromaFormatIdc == 2 && !s.SeparateColourPlaneFlag:
subHeightC = 1

case s.ChromaFormatIdc == 3 && !s.SeparateColourPlaneFlag:
subHeightC = 1
}

var frameMbsOnlyFlagUint32 uint32
if s.FrameMbsOnlyFlag {
f = 1
frameMbsOnlyFlagUint32 = 1
}

var chromaArrayType uint32
if !s.SeparateColourPlaneFlag {
chromaArrayType = s.ChromaFormatIdc
} else {
chromaArrayType = 0
}

var cropUnitY uint32
if chromaArrayType == 0 {
cropUnitY = 2 - frameMbsOnlyFlagUint32
} else {
cropUnitY = subHeightC * (2 - frameMbsOnlyFlagUint32)
}

frameHeightInMbs := (2 - frameMbsOnlyFlagUint32) * (s.PicHeightInMapUnitsMinus1 + 1)

if s.FrameCropping != nil {
return int(((2 - f) * (s.PicHeightInMapUnitsMinus1 + 1) * 16) -
(s.FrameCropping.TopOffset+s.FrameCropping.BottomOffset)*2)
return int(16*frameHeightInMbs - cropUnitY*(s.FrameCropping.TopOffset+s.FrameCropping.BottomOffset))
}

return int((2 - f) * (s.PicHeightInMapUnitsMinus1 + 1) * 16)
picHeightInMbs := frameHeightInMbs // / (1 + s.FieldPicFlag)
picHeightInSamplesL := picHeightInMbs * 16

return int(picHeightInSamplesL)
}

// FPS returns the frames per second of the video.
Expand Down
75 changes: 65 additions & 10 deletions pkg/codecs/h264/sps_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 12,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxFrameNumMinus4: 6,
PicOrderCntType: 2,
MaxNumRefFrames: 1,
Expand Down Expand Up @@ -57,7 +57,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 31,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxPicOrderCntLsbMinus4: 2,
MaxNumRefFrames: 4,
PicWidthInMbsMinus1: 79,
Expand Down Expand Up @@ -122,7 +122,7 @@ func TestSPSUnmarshal(t *testing.T) {
},
},
1920,
1080,
1084,
30,
},
{
Expand All @@ -136,7 +136,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 40,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxPicOrderCntLsbMinus4: 2,
MaxNumRefFrames: 4,
PicWidthInMbsMinus1: 119,
Expand Down Expand Up @@ -175,7 +175,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 41,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxFrameNumMinus4: 8,
Log2MaxPicOrderCntLsbMinus4: 5,
MaxNumRefFrames: 4,
Expand Down Expand Up @@ -206,7 +206,7 @@ func TestSPSUnmarshal(t *testing.T) {
},
},
1920,
1084,
1080,
25,
},
{
Expand All @@ -215,7 +215,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 32,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxPicOrderCntLsbMinus4: 4,
MaxNumRefFrames: 1,
PicWidthInMbsMinus1: 79,
Expand Down Expand Up @@ -245,7 +245,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 50,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
ScalingList4x4: [][]int32{
{
16, 16, 16, 16, 16, 16, 16, 16,
Expand Down Expand Up @@ -314,7 +314,7 @@ func TestSPSUnmarshal(t *testing.T) {
SPS{
ProfileIdc: 100,
LevelIdc: 42,
ChromeFormatIdc: 1,
ChromaFormatIdc: 1,
Log2MaxFrameNumMinus4: 4,
Log2MaxPicOrderCntLsbMinus4: 4,
MaxNumRefFrames: 2,
Expand Down Expand Up @@ -420,7 +420,62 @@ func TestSPSUnmarshal(t *testing.T) {
},
},
1920,
1080,
1084,
25,
},
{
"1920x1080 mbs_only_flag = 0",
[]byte{
0x67, 0x4d, 0x40, 0x28, 0xab, 0x60, 0x3c, 0x02,
0x23, 0xef, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00,
0x10, 0x00, 0x00, 0x03, 0x03, 0x2e, 0x94, 0x00,
0x35, 0x64, 0x06, 0xb2, 0x85, 0x08, 0x0e, 0xe2,
0xc5, 0x22, 0xc0,
},
SPS{
ProfileIdc: 77,
ConstraintSet1Flag: true,
LevelIdc: 40,
Log2MaxFrameNumMinus4: 1,
Log2MaxPicOrderCntLsbMinus4: 2,
MaxNumRefFrames: 2,
PicWidthInMbsMinus1: 119,
PicHeightInMapUnitsMinus1: 33,
Direct8x8InferenceFlag: true,
FrameCropping: &SPS_FrameCropping{
BottomOffset: 2,
},
VUI: &SPS_VUI{
AspectRatioInfoPresentFlag: true,
AspectRatioIdc: 1,
TimingInfo: &SPS_TimingInfo{
NumUnitsInTick: 1,
TimeScale: 50,
FixedFrameRateFlag: true,
},
NalHRD: &SPS_HRD{
BitRateScale: 4,
CpbSizeScale: 10,
BitRateValueMinus1: []uint32{3416},
CpbSizeValueMinus1: []uint32{213},
CbrFlag: []bool{false},
InitialCpbRemovalDelayLengthMinus1: 20,
CpbRemovalDelayLengthMinus1: 5,
DpbOutputDelayLengthMinus1: 1,
},
PicStructPresentFlag: true,
BitstreamRestriction: &SPS_BitstreamRestriction{
MotionVectorsOverPicBoundariesFlag: true,
MaxBytesPerPicDenom: 2,
Log2MaxMvLengthHorizontal: 10,
Log2MaxMvLengthVertical: 9,
MaxNumReorderFrames: 1,
MaxDecFrameBuffering: 4,
},
},
},
1920,
1084,
25,
},
} {
Expand Down
8 changes: 4 additions & 4 deletions pkg/formats/fmp4/init_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -390,7 +390,7 @@ var casesInit = []struct {
0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
0x07, 0x80, 0x00, 0x00, 0x04, 0x38, 0x00, 0x00,
0x07, 0x80, 0x00, 0x00, 0x04, 0x3c, 0x00, 0x00,
0x00, 0x00, 0x01, 0x88,
'm', 'd', 'i', 'a',
0x00, 0x00, 0x00, 0x20,
Expand Down Expand Up @@ -425,7 +425,7 @@ var casesInit = []struct {
0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x80, 0x04,
0x38, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00,
0x3c, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand Down Expand Up @@ -708,7 +708,7 @@ var casesInit = []struct {
0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00,
0x07, 0x80, 0x00, 0x00, 0x04, 0x38, 0x00, 0x00,
0x07, 0x80, 0x00, 0x00, 0x04, 0x3c, 0x00, 0x00,
0x00, 0x00, 0x01, 0x88,
'm', 'd', 'i', 'a',
0x00, 0x00, 0x00, 0x20,
Expand Down Expand Up @@ -741,7 +741,7 @@ var casesInit = []struct {
0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0x80, 0x04,
0x38, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00,
0x3c, 0x00, 0x48, 0x00, 0x00, 0x00, 0x48, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
Expand Down

0 comments on commit 328cb87

Please sign in to comment.