PR #9380 from Avishag: Y411 as NV12; OpenGL processing

common/model-views.cpp

@@ -1001,13 +1001,15 @@ namespace rs2
         streaming(false), _pause(false),
         depth_colorizer(std::make_shared<rs2::gl::colorizer>()),
         yuy2rgb(std::make_shared<rs2::gl::yuy_decoder>()),
+        y411(std::make_shared<rs2::gl::y411_decoder>()),
         depth_decoder(std::make_shared<rs2::depth_huffman_decoder>()),
         viewer(viewer),
         detected_objects(device_detected_objects)
     {
         supported_options = s->get_supported_options();
         restore_processing_block("colorizer", depth_colorizer);
         restore_processing_block("yuy2rgb", yuy2rgb);
+        restore_processing_block("y411", y411);
 
         std::string device_name(dev.get_info(RS2_CAMERA_INFO_NAME));
         std::string sensor_name(s->get_info(RS2_CAMERA_INFO_NAME));
@@ -2074,6 +2076,7 @@ namespace rs2
 
             save_processing_block_to_config_file("colorizer", depth_colorizer);
             save_processing_block_to_config_file("yuy2rgb", yuy2rgb);
+            save_processing_block_to_config_file("y411", y411);
 
             for (auto&& pbm : post_processing) pbm->save_to_config_file();
         }
@@ -2310,6 +2313,7 @@ namespace rs2
         profile = p;
         texture->colorize = d->depth_colorizer;
         texture->yuy2rgb = d->yuy2rgb;
+        texture->y411 = d->y411;
         texture->depth_decode = d->depth_decoder;
 
         if (auto vd = p.as<video_stream_profile>())

common/model-views.h

@@ -670,6 +670,7 @@ namespace rs2
 
         std::shared_ptr<rs2::colorizer> depth_colorizer;
         std::shared_ptr<rs2::yuy_decoder> yuy2rgb;
+        std::shared_ptr<rs2::y411_decoder> y411;
         std::shared_ptr<processing_block_model> zero_order_artifact_fix;
         std::shared_ptr<rs2::depth_huffman_decoder> depth_decoder;
 

common/rendering.h

@@ -866,6 +866,7 @@ namespace rs2
     public:
         std::shared_ptr<colorizer> colorize;
         std::shared_ptr<yuy_decoder> yuy2rgb;
+        std::shared_ptr<y411_decoder> y411;
         std::shared_ptr<depth_huffman_decoder> depth_decode;
         bool zoom_preview = false;
         rect curr_preview_rect{};
@@ -1061,6 +1062,30 @@ namespace rs2
                         glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, data);
                     }
                     break;
+                case RS2_FORMAT_Y411:
+                    if (y411)
+                    {
+                        if (auto colorized_frame = y411->process(frame).as<video_frame>())
+                        {
+                            if (!colorized_frame.is<gl::gpu_frame>())
+                            {
+                                glBindTexture(GL_TEXTURE_2D, texture);
+                                data = colorized_frame.get_data();
+
+                                glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB,
+                                    colorized_frame.get_width(),
+                                    colorized_frame.get_height(),
+                                    0, GL_RGB, GL_UNSIGNED_BYTE,
+                                    colorized_frame.get_data());
+                            }
+                            rendered_frame = colorized_frame;
+                        }
+                    }
+                    else
+                    {
+                        glTexImage2D(GL_TEXTURE_2D, 0, GL_RGB, width, height, 0, GL_LUMINANCE_ALPHA, GL_UNSIGNED_BYTE, data);
+                    }
+                    break;
                 case RS2_FORMAT_UYVY: // Use luminance component only to avoid costly UVUY->RGB conversion
                     glTexImage2D(GL_TEXTURE_2D, 0, GL_LUMINANCE, width, height, 0, GL_LUMINANCE, GL_UNSIGNED_SHORT, data);
                     break;

include/librealsense2-gl/rs_processing_gl.h

@@ -75,6 +75,17 @@ struct glfw_binding
 */
 rs2_processing_block* rs2_gl_create_yuy_decoder(int api_version, rs2_error** error);
 
+/**
+* Creates y411 decoder processing block. This block accepts raw y411 frames and outputs frames in RGB8.
+*     https://www.fourcc.org/pixel-format/yuv-y411/
+* Y411 is disguised as NV12 to allow Linux compatibility. Both are 12bpp encodings that allow high-resolution
+* modes in the camera to still fit within the USB3 limits (YUY wasn't enough).
+*
+* \param[in] api_version Users are expected to pass their version of \c RS2_API_VERSION to make sure they are running the correct librealsense version.
+* \param[out] error  if non-null, receives any error that occurs during this call, otherwise, errors are ignored
+*/
+rs2_processing_block* rs2_gl_create_y411_decoder(int api_version, rs2_error** error);
+
 /**
 * Sets new value to one of processing blocks matrices
 * \param[in] block      Processing block object

include/librealsense2-gl/rs_processing_gl.hpp

@@ -138,6 +138,27 @@ namespace rs2
             }
         };
 
+        /**
+         * Similar in functionality (Y411->RGB8 conversion) to rs2::y411_decoder but performed on the GPU.
+         */
+        class y411_decoder : public rs2::y411_decoder
+        {
+        public:
+            y411_decoder() : rs2::y411_decoder( init() ) {}
+
+        private:
+            static std::shared_ptr<rs2_processing_block> init()
+            {
+                rs2_error* e = nullptr;
+                auto block = std::shared_ptr<rs2_processing_block>(
+                    rs2_gl_create_y411_decoder(RS2_API_VERSION, &e),
+                    rs2_delete_processing_block);
+                error::handle(e);
+
+                return block;
+            }
+        };
+
         /**
         * Colorizer can be used for Depth->RGB conversion, including histogram equalization
         * Similar in functionality to rs2::colorizer but performed on the GPU

include/librealsense2/h/rs_processing.h

@@ -53,6 +53,19 @@ rs2_processing_block* rs2_create_pointcloud(rs2_error** error);
 */
 rs2_processing_block* rs2_create_yuy_decoder(rs2_error** error);
 
+/**
+* Creates y411 decoder processing block. This block accepts raw y411 frames and outputs frames in RGB8.
+*     https://www.fourcc.org/pixel-format/yuv-y411/
+* Y411 is disguised as NV12 to allow Linux compatibility. Both are 12bpp encodings that allow high-resolution
+* modes in the camera to still fit within the USB3 limits (YUY wasn't enough).
+*
+* The SDK will automatically try to use SSE2 and AVX instructions and CUDA where available to get
+* best performance. Other implementations (using GLSL, OpenCL, Neon and NCS) should follow.
+*
+* \param[out] error  if non-null, receives any error that occurs during this call, otherwise, errors are ignored
+*/
+rs2_processing_block* rs2_create_y411_decoder(rs2_error** error);
+
 /**
 * Creates depth thresholding processing block
 * By controlling min and max options on the block, one could filter out depth values

include/librealsense2/hpp/rs_processing.hpp

@@ -508,6 +508,32 @@ namespace rs2
         }
     };
 
+    class y411_decoder : public filter
+    {
+    public:
+        /**
+         * Creates y411 decoder processing block. This block accepts raw y411 frames and outputs frames in RGB8.
+         *     https://www.fourcc.org/pixel-format/yuv-y411/
+         * Y411 is disguised as NV12 to allow Linux compatibility. Both are 12bpp encodings that allow high-resolution
+         * modes in the camera to still fit within the USB3 limits (YUY wasn't enough).
+         */
+        y411_decoder() : filter(init()) { }
+
+    protected:
+        y411_decoder(std::shared_ptr<rs2_processing_block> block) : filter(block) {}
+
+    private:
+        static std::shared_ptr<rs2_processing_block> init()
+        {
+            rs2_error* e = nullptr;
+            auto block = std::shared_ptr<rs2_processing_block>(
+                rs2_create_y411_decoder(&e),
+                rs2_delete_processing_block);
+            error::handle(e);
+
+            return block;
+        }
+    };
   class threshold_filter : public filter
     {
     public:

src/gl/CMakeLists.txt

@@ -10,6 +10,8 @@ set(REALSENSE_GL_CPP
     synthetic-stream-gl.cpp
     yuy2rgb-gl.h
     yuy2rgb-gl.cpp
+    y4112rgb-gl.h
+    y4112rgb-gl.cpp
     pointcloud-gl.h
     pointcloud-gl.cpp
     rs-gl.cpp

src/gl/realsense-gl.def

@@ -6,6 +6,7 @@ EXPORTS
     rs2_gl_create_pointcloud
     rs2_gl_create_camera_renderer
     rs2_gl_create_yuy_decoder
+    rs2_gl_create_y411_decoder
     rs2_gl_create_uploader
     rs2_gl_create_colorizer
     rs2_gl_create_pointcloud_renderer

src/gl/rs-gl.cpp

@@ -4,6 +4,7 @@
 #include "api.h"
 #include "synthetic-stream-gl.h"
 #include "yuy2rgb-gl.h"
+#include "y4112rgb-gl.h"
 #include "align-gl.h"
 #include "pointcloud-gl.h"
 #include "../include/librealsense2/h/rs_types.h"
@@ -13,6 +14,7 @@
 #include "pc-shader.h"
 #include "colorizer-gl.h"
 #include "proc/color-formats-converter.h"
+#include "proc/y411-converter.h"
 #include "proc/colorizer.h"
 #include "proc/align.h"
 #include "log.h"
@@ -79,6 +81,19 @@ rs2_processing_block* rs2_gl_create_yuy_decoder(int api_version, rs2_error** err
 }
 NOARGS_HANDLE_EXCEPTIONS_AND_RETURN(nullptr)
 
+rs2_processing_block* rs2_gl_create_y411_decoder(int api_version, rs2_error** error) BEGIN_API_CALL
+{
+    verify_version_compatibility(api_version);
+
+    auto block = std::make_shared<librealsense::gl::y411_2rgb>();
+    auto backup = std::make_shared<librealsense::y411_converter>(RS2_FORMAT_RGB8);
+    auto dual = std::make_shared<librealsense::gl::dual_processing_block>();
+    dual->add(block);
+    dual->add(backup);
+    return new rs2_processing_block{ dual };
+}
+NOARGS_HANDLE_EXCEPTIONS_AND_RETURN(nullptr)
+
 unsigned int rs2_gl_frame_get_texture_id(const rs2_frame* frame_ref, unsigned int id, rs2_error** error) BEGIN_API_CALL
 {
     VALIDATE_NOT_NULL(frame_ref);