Additional testing and associated bug fixes related to H.265 payloader

codecs/h265_packet.go

@@ -962,7 +962,7 @@ type H265Payloader struct {
 // Payload fragments a H265 packet across one or more byte arrays
 func (p *H265Payloader) Payload(mtu uint16, payload []byte) [][]byte { //nolint: gocognit
 	var payloads [][]byte
-	if len(payload) == 0 {
+	if len(payload) == 0 || mtu == 0 {
 		return payloads
 	}
 
@@ -998,7 +998,7 @@ func (p *H265Payloader) Payload(mtu uint16, payload []byte) [][]byte { //nolint:
 			}
 		} else {
 			// construct an aggregation packet
-			aggregationPacketSize := aggregationBufferSize + 2
+			aggregationPacketSize := aggregationBufferSize
 			buf := make([]byte, aggregationPacketSize)
 
 			layerID := uint8(math.MaxUint8)
@@ -1039,22 +1039,39 @@ func (p *H265Payloader) Payload(mtu uint16, payload []byte) [][]byte { //nolint:
 		aggregationBufferSize = 0
 	}
 
+	calcMarginalAggregationSize := func(nalu []byte) int {
+		marginalAggregationSize := len(nalu) + 2 // +2 is NALU size Field size
+		if len(bufferedNALUs) == 1 {
+			marginalAggregationSize = len(nalu) + 4 // +4 are Aggregation header + NALU size Field size
+		}
+		if p.AddDONL {
+			if len(bufferedNALUs) == 0 {
+				marginalAggregationSize += 2
+			} else {
+				marginalAggregationSize++
+			}
+		}
+		return marginalAggregationSize
+	}
+
 	emitNalus(payload, func(nalu []byte) {
-		if len(nalu) == 0 {
+		if len(nalu) < 2 {
+			// NALU header is 2 bytes
 			return
 		}
 
-		if len(nalu) <= int(mtu) {
+		naluLen := len(nalu) + 2
+		if p.AddDONL {
+			naluLen += 2
+		}
+		if naluLen <= int(mtu) {
 			// this nalu fits into a single packet, either it can be emitted as
 			// a single nalu or appended to the previous aggregation packet
-
-			marginalAggregationSize := len(nalu) + 2
-			if p.AddDONL {
-				marginalAggregationSize++
-			}
+			marginalAggregationSize := calcMarginalAggregationSize(nalu)
 
 			if aggregationBufferSize+marginalAggregationSize > int(mtu) {
 				flushBufferedNals()
+				marginalAggregationSize = calcMarginalAggregationSize(nalu)
 			}
 			bufferedNALUs = append(bufferedNALUs, nalu)
 			aggregationBufferSize += marginalAggregationSize
@@ -1077,7 +1094,7 @@ func (p *H265Payloader) Payload(mtu uint16, payload []byte) [][]byte { //nolint:
 			// the nalu header is omitted from the fragmentation packet payload
 			nalu = nalu[h265NaluHeaderSize:]
 
-			if maxFUPayloadSize == 0 || len(nalu) == 0 {
+			if maxFUPayloadSize <= 0 || len(nalu) == 0 {
 				return
 			}