Improvements to JPEG depay component.

- make sure SDP media corresponds to JPEG - handle default width/height if available, otherwise use the JPEG header in the RTP packets - add a link to the JPEG RTP spec to help understand what the headers code does
AxisCommunications · Apr 11, 2018 · b69ec15 · b69ec15
1 parent 2fc25ed
commit b69ec15
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 24 deletions.
diff --git a/lib/components/jpegdepay/headers.js b/lib/components/jpegdepay/headers.js
@@ -1,3 +1,13 @@
+
+/**
+ * Generate frame and scan headers that can be prepended to the
+ * RTP/JPEG data payload to produce a JPEG compressed image in
+ * interchange format.
+ *
+ * For detailed information, check Appendix A of:
+ * https://tools.ietf.org/html/rfc2435
+ */
+
 function makeImageHeader () {
   return Buffer.from([0xff, 0xd8])
 }
@@ -20,7 +30,7 @@ function makeQuantHeader (precision, qTable) {
 
 function makeFrameHeader (width, height, type) {
   return Buffer.from([
-    0xff, 0xc0, // SOF_0
+    0xff, 0xc0, // SOF_0 (Start Of Frame)
     0,
     17,
     8,
@@ -145,7 +155,7 @@ function makeHuffmanHeader () {
 
 function makeScanHeader () {
   return Buffer.from([
-    0xff, 0xda,
+    0xff, 0xda, // SOS (Start Of Scan)
     0,
     12,
     3,

diff --git a/lib/components/jpegdepay/jpegdepay-component.js b/lib/components/jpegdepay/jpegdepay-component.js
@@ -14,14 +14,30 @@ class JPEGDepayComponent extends Component {
       objectMode: true,
       transform (msg, encoding, callback) {
         if (msg.type === SDP) {
-          payloadType = Number(msg.sdp.media[0].rtpmap.payloadType)
-          const framesize = msg.sdp.media[0].framesize
-          const [width, height] = framesize.split(' ')[1].split('-').map(Number)
-          msg.framesize = {width, height}
-
-          console.log(width, height)
-
-          jpegDepay = jpegDepayFactory(width, height)
+          const jpegMedia = msg.sdp.media.find(media => {
+            return (
+              media.type === 'video' &&
+              media.rtpmap &&
+              media.rtpmap.encodingName === 'JPEG'
+            )
+          })
+          if (jpegMedia) {
+            payloadType = Number(jpegMedia.rtpmap.payloadType)
+            const framesize = jpegMedia.framesize
+            // `framesize` is an SDP field that is present in e.g. Axis camera's
+            // and is used because the width and height that can be sent inside
+            // the JPEG header are both limited to 2040.
+            // If present, we use this width and height as the default values
+            // to be used by the jpeg depay function, otherwise we ignore this
+            // and let the JPEG header inside the RTP packets determine this.
+            if (framesize) {
+              const [width, height] = framesize.split(' ')[1].split('-').map(Number)
+              msg.framesize = {width, height}
+              jpegDepay = jpegDepayFactory(width, height)
+            } else {
+              jpegDepay = jpegDepayFactory()
+            }
+          }
 
           callback(null, msg)
         } else if (msg.type === RTP && Rtp.payloadType(msg.data) === payloadType) {
@@ -33,10 +49,11 @@ class JPEGDepayComponent extends Component {
           if (Rtp.marker(msg.data) && packets.length > 0) {
             const jpegFrame = jpegDepay(packets)
             this.push({
-              data: jpegFrame,
               timestamp: Rtp.timestamp(msg.data),
               ntpTimestamp: msg.ntpTimestamp,
               payloadType: Rtp.payloadType(msg.data),
+              data: jpegFrame.data,
+              size: jpegFrame.size,
               type: JPEG
             })
             packets = []

diff --git a/lib/components/jpegdepay/jpegdepay.js b/lib/components/jpegdepay/jpegdepay.js
@@ -40,7 +40,7 @@ const {
    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
  */
 
-module.exports = function jpegDepayFactory (width, height) {
+module.exports = function jpegDepayFactory (defaultWidth = 0, defaultHeight = 0) {
   const IMAGE_HEADER = makeImageHeader()
   const HUFFMAN_HEADER = makeHuffmanHeader()
   const SCAN_HEADER = makeScanHeader()
@@ -55,8 +55,8 @@ module.exports = function jpegDepayFactory (width, height) {
       const fragmentOffset = payload.readUInt8(1) << 16 | payload.readUInt8(2) << 8 | payload.readUInt8(3)
       const type = payload.readUInt8(4)
       const Q = payload.readUInt8(5)
-      // const width = payload.readUInt8(6) * 8
-      // const height = payload.readUInt8(7) * 8
+      const width = payload.readUInt8(6) * 8 || defaultWidth
+      const height = payload.readUInt8(7) * 8 || defaultHeight
       payload = payload.slice(8)
 
       // Parse and extract Restart Marker header if present.
@@ -81,23 +81,26 @@ module.exports = function jpegDepayFactory (width, height) {
       return payload
     })
 
-    const { precision, qTable, type } = metadata
+    const { precision, qTable, type, width, height } = metadata
 
     const quantHeader = makeQuantHeader(precision, qTable)
 
     if (metadata.DRI !== 0) {
-      // makeDRIHeader
+      throw new Error('not implemented: DRI')
     }
 
     const frameHeader = makeFrameHeader(width, height, type)
 
-    return Buffer.concat([
-      IMAGE_HEADER,
-      quantHeader,
-      frameHeader,
-      HUFFMAN_HEADER,
-      SCAN_HEADER,
-      ...fragments
-    ])
+    return {
+      size: {width, height},
+      data: Buffer.concat([
+        IMAGE_HEADER,
+        quantHeader,
+        frameHeader,
+        HUFFMAN_HEADER,
+        SCAN_HEADER,
+        ...fragments
+      ])
+    }
   }
 }