Fix: depth images with video now save as greyscale instead of being g…

…arbled. This fixes issue #7. New API call: MissionSpec::requestVideoWithDepth.

Malmo/src/MissionRecordSpec.h

@@ -34,7 +34,7 @@ namespace malmo
 
         //! Requests that video be recorded, at the specified quality.
         //! Ensure that the width of the video requested is divisible by 4, and the height of the video requested is divisible by 2.
-        //! \param frames_per_second The number of frames to record per second
+        //! \param frames_per_second The number of frames to record per second. e.g. 20.
         //! \param bit_rate The bit rate to record at. e.g. 400000 for 400kbps.
         void recordMP4(int frames_per_second, int64_t bit_rate);
 

Malmo/src/MissionSpec.cpp

@@ -217,6 +217,16 @@ namespace malmo
         vps.set( VideoProducer( width, height ) );
     }
 
+    void MissionSpec::requestVideoWithDepth(int width, int height)
+    {
+        AgentHandlers::VideoProducer_optional& vps = this->mission->AgentSection().front().AgentHandlers().VideoProducer();
+        if (vps.present())
+            throw runtime_error("MissionSpec::requestVideoWithDepth : video was already requested for this agent");
+        VideoProducer vp(width, height);
+        vp.want_depth(true);
+        vps.set(vp);
+    }
+
     void MissionSpec::rewardForReachingPosition(int x, int y, int z, float amount, float tolerance)
     {
         AgentHandlers::RewardForReachingPosition_optional& rrp = this->mission->AgentSection().front().AgentHandlers().RewardForReachingPosition();

Malmo/src/MissionSpec.h

@@ -116,10 +116,18 @@ namespace malmo
             void setModeToSpectator();
 
             //! Asks for image data to be sent from Minecraft for the agent. Only supports single agent missions.
+            //! Data will be delivered in a TimestampedVideoFrame structure as RGBRGBRGB...
             //! \param width The width of the image in pixels. Ensure this is divisible by 4.
             //! \param height The height of the image in pixels. Ensure this is divisible by 2.
             void requestVideo(int width, int height);
 
+            //! Asks for image data and depth data to be sent from Minecraft for the agent. Only supports single agent missions.
+            //! Data will be delivered in a TimestampedVideoFrame structure as RGBDRGBDRGBD...
+            //! If saving the video to file only the depth will be recorded, as greyscale.
+            //! \param width The width of the image in pixels. Ensure this is divisible by 4.
+            //! \param height The height of the image in pixels. Ensure this is divisible by 2.
+            void requestVideoWithDepth(int width, int height);
+
             //! Asks for a reward to be sent to the agent when it reaches a certain position. Only supports single agent missions.
             //! \param x The east-west location.
             //! \param y The up-down location.

Malmo/src/PosixFrameWriter.cpp

@@ -134,15 +134,15 @@ namespace malmo
         }
     }
 
-    void PosixFrameWriter::doWrite(TimestampedVideoFrame message, int frame_index)
+    void PosixFrameWriter::doWrite(char* rgb, int width, int height, int frame_index)
     {
         std::ostringstream oss;
-        oss << "P6\n" << message.width << " " << message.height << "\n255\n";
+        oss << "P6\n" << width << " " << height << "\n255\n";
         ssize_t ret = ::write( this->pipe_fd[1], oss.str().c_str(), oss.str().size() );
         if( ret < 0 )
             throw std::runtime_error( "Call to write failed." );
 
-        ret = ::write( this->pipe_fd[1], (char *)&message.pixels[0], message.pixels.size() );
+        ret = ::write( this->pipe_fd[1], rgb, width*height*3 );
         if( ret < 0 )
             throw std::runtime_error( "Call to write failed." );
     }

Malmo/src/PosixFrameWriter.h

@@ -18,9 +18,9 @@ namespace malmo
         ~PosixFrameWriter();
         void open() override;
         void close() override;
-        void doWrite(TimestampedVideoFrame message, int frame_index) override;
 
     private:
+        void doWrite(char* rgb, int width, int height, int frame_index) override;
         std::string search_path();
 
         int64_t bit_rate;

Malmo/src/TimestampedVideoFrame.h

@@ -33,7 +33,7 @@ namespace malmo
         //! The height of the image in pixels.
         short height;
 
-        //! The number of channels. e.g. 3 for RGB data, 4 for RGBA
+        //! The number of channels. e.g. 3 for RGB data, 4 for RGBD
         short channels;
 
         //! The pixels, stored as channels then columns then rows. Length should be width*height*channels.

Malmo/src/VideoFrameWriter.cpp

@@ -113,7 +113,29 @@ namespace malmo
                     break;
                 }
 
-                this->doWrite(frame, count);
+                if (frame.channels == 4)
+                {
+                    // extract DDD from RGBD
+                    // TODO: support other options, output multiple videos
+                    const DWORD num_out_bytes = frame.width * frame.height * 3;
+                    char *out_pixels = new char[num_out_bytes];
+                    int j = 0;
+                    for (int i = 0; i < frame.width*frame.height; i++)
+                    {
+                        out_pixels[j++] = frame.pixels[i * 4 + 3];
+                        out_pixels[j++] = frame.pixels[i * 4 + 3];
+                        out_pixels[j++] = frame.pixels[i * 4 + 3];
+                    }
+                    this->doWrite(out_pixels, frame.width, frame.height, count);
+
+                    delete[]out_pixels;
+                }
+                else if (frame.channels == 3)
+                {
+                    // write the RGB data directly
+                    this->doWrite((char*)&frame.pixels[0], frame.width, frame.height, count);
+                }
+                else throw std::runtime_error("Unsupported number of channels");
 
                 count++;
             }

Malmo/src/VideoFrameWriter.h

@@ -28,14 +28,15 @@ namespace malmo
         virtual ~VideoFrameWriter();
         virtual void open();
         virtual void close();
-        virtual void doWrite(TimestampedVideoFrame frame, int frame_index) = 0;
 
         void write(TimestampedVideoFrame frame);
         bool isOpen() const;
 
         static std::unique_ptr<VideoFrameWriter> create(std::string path, short width, short height, int frames_per_second, int64_t bit_rate);
 
     protected:
+        virtual void doWrite(char* rgb, int width, int height, int frame_index) = 0;
+
         std::string path;
         short width;
         short height;

Malmo/src/WindowsFrameWriter.cpp

@@ -82,20 +82,20 @@ namespace malmo
         }
     }
 
-    void WindowsFrameWriter::doWrite(TimestampedVideoFrame message, int frame_index)
+    void WindowsFrameWriter::doWrite(char* rgb, int width, int height, int frame_index)
     {
         DWORD dwWritten;
         BOOL bSuccess = FALSE;
         std::stringstream header;
-        header << "P6\n" << message.width << " " << message.height << "\n255\n";
+        header << "P6\n" << width << " " << height << "\n255\n";
         std::string headerStr = header.str();        
 
         bSuccess = WriteFile(this->g_hChildStd_IN_Wr, headerStr.c_str(), (DWORD)headerStr.length(), &dwWritten, NULL);
         if (!bSuccess){
             throw std::runtime_error("Unable to write frame header to pipe.");
         }
 
-        bSuccess = WriteFile(this->g_hChildStd_IN_Wr, (char *)&message.pixels[0], (DWORD)message.pixels.size(), &dwWritten, NULL);        
+        bSuccess = WriteFile(this->g_hChildStd_IN_Wr, rgb, width*height*3, &dwWritten, NULL);
         if (!bSuccess){
             throw std::runtime_error("Unable to write frame pixels to pipe.");
         }

Malmo/src/WindowsFrameWriter.h

@@ -28,9 +28,9 @@ namespace malmo
         ~WindowsFrameWriter();
         void open() override;
         void close() override;
-        void doWrite(TimestampedVideoFrame message, int frame_index) override;
 
     private:
+        void doWrite(char* rgb, int width, int height, int frame_index) override;
         void runFFMPEG();
         std::string search_path();