Skip to content

Commit

Permalink
Metadata fields parsing and extraction
Browse files Browse the repository at this point in the history 
cpp-config-ui to provide metadata widget for selected streams
cpp-headless dumps metadata to csv file
Improve Ubuntu kernel patch robustness
Handle Fisheye auto-exposure as meta-data attribute
Metadata API verification to be covered by unit-test
Update python bindings
Fix for Metadata playback

Tracked On: DSO-3542

Change-Id: I8b993a7e93291b7246984529bdf0042d7c9e6f7e
  • Loading branch information
@ev-mp
ev-mp committed May 24, 2017
1 parent edffacf commit 0862cd0
Show file tree
Hide file tree
Showing 36 changed files with 1,140 additions and 169 deletions.
4 changes: 3 additions & 1 deletion CMake/realsense.def
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,14 @@ EXPORTS
rs2_distortion_to_string
rs2_option_to_string
rs2_camera_info_to_string
rs2_frame_metadata_to_string
rs2_timestamp_domain_to_string
rs2_visual_preset_to_string
rs2_notification_category_to_string

rs2_log_to_console
rs2_log_to_file

rs2_create_syncer
rs2_start_syncer
rs2_wait_for_frames
Expand Down
6 changes: 4 additions & 2 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@ set(REALSENSE_HPP
src/subdevice.h
src/algo.h
src/option.h
src/metadata.h
src/metadata-parser.h
src/error-handling.h
src/hw-monitor.h
src/image.h
Expand Down Expand Up @@ -226,7 +228,7 @@ if(WIN32)
src/sr300.h
src/ivcam-private.h
)

foreach(flag_var
CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELEASE
CMAKE_CXX_FLAGS_MINSIZEREL CMAKE_CXX_FLAGS_RELWITHDEBINFO
Expand Down Expand Up @@ -274,7 +276,7 @@ elseif(UNIX)
find_package(PkgConfig REQUIRED)
find_package (Threads REQUIRED)
list(APPEND librealsense_PKG_DEPS "Threads")

pkg_search_module(LIBUSB1 REQUIRED libusb-1.0)
if(LIBUSB1_FOUND)
include_directories(SYSTEM ${LIBUSB1_INCLUDE_DIRS})
Expand Down
4 changes: 2 additions & 2 deletions bindings/pybackend_extras.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ namespace pybackend2 {
const uint16_t pre_header_data = 0xcdab;
raw_data.resize(HW_MONITOR_BUFFER_SIZE);
auto write_ptr = raw_data.data();
auto header_size = 4;
size_t header_size = 4;

auto cur_index = 2;
size_t cur_index = 2;
*reinterpret_cast<uint16_t *>(write_ptr + cur_index) = pre_header_data;
cur_index += sizeof(uint16_t);
*reinterpret_cast<unsigned int *>(write_ptr + cur_index) = cmd_op_code;
Expand Down
8 changes: 7 additions & 1 deletion bindings/python.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,13 @@ PYBIND11_PLUGIN(NAME) {
.value("count", RS2_TIMESTAMP_DOMAIN_COUNT);

py::enum_<rs2_frame_metadata> frame_metadata(m, "frame_metadata");
frame_metadata.value("actual_exposure", RS2_FRAME_METADATA_ACTUAL_EXPOSURE)
frame_metadata.value("frame counter", RS2_FRAME_METADATA_FRAME_COUNTER)
.value("frame_timestamp_usec", RS2_FRAME_METADATA_FRAME_TIMESTAMP)
.value("sensor_timestamp_usec", RS2_FRAME_METADATA_SENSOR_TIMESTAMP)
.value("actual_exposure_usec", RS2_FRAME_METADATA_ACTUAL_EXPOSURE)
.value("gain_level", RS2_FRAME_METADATA_GAIN_LEVEL)
.value("auto_exposure", RS2_FRAME_METADATA_AUTO_EXPOSURE)
.value("white_balance_temp_k", RS2_FRAME_METADATA_WHITE_BALANCE)
.value("count", RS2_FRAME_METADATA_COUNT);

py::class_<rs2::frame> frame(m, "frame");
Expand Down
116 changes: 100 additions & 16 deletions examples/cpp-config-ui.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
#include <iomanip>
#include <map>
#include <sstream>
#include <array>
#include <mutex>

#pragma comment(lib, "opengl32.lib")
Expand Down Expand Up @@ -807,20 +808,30 @@ class fps_calc

typedef std::map<rs2_stream, rect> streams_layout;

struct frame_metadata
{
std::array<std::pair<bool,rs2_metadata_t>,RS2_FRAME_METADATA_COUNT> md_attributes{};
};

class stream_model
{
public:
rect layout;
texture_buffer texture;
float2 size;
rs2_format format;
rect layout;
texture_buffer texture;
float2 size;
rs2_stream stream;
rs2_format format;
std::chrono::high_resolution_clock::time_point last_frame;
double timestamp;
unsigned long long frame_number;
double timestamp;
unsigned long long frame_number;
rs2_timestamp_domain timestamp_domain;
fps_calc fps;
rect roi_display_rect;
bool capturing_roi = false;
fps_calc fps;
rect roi_display_rect{};
frame_metadata frame_md;
bool metadata_displayed = false;
bool roi_supported = false; // supported by device in its current state
bool roi_checked = false; // actively selected by user
bool capturing_roi = false; // active modification of roi
std::shared_ptr<subdevice_model> dev;

void upload_frame(frame&& f)
Expand All @@ -832,12 +843,24 @@ class stream_model
auto height = (is_motion) ? 480.f : f.get_height();

size = { static_cast<float>(width), static_cast<float>(height)};
stream = f.get_stream_type();
format = f.get_format();
frame_number = f.get_frame_number();
timestamp_domain = f.get_frame_timestamp_domain();
timestamp = f.get_timestamp();
fps.add_timestamp(f.get_timestamp(), f.get_frame_number());

// populate frame metadata attributes
for (auto i=0; i< RS2_FRAME_METADATA_COUNT; i++)
{
if (f.supports_frame_metadata((rs2_frame_metadata)i))
frame_md.md_attributes[i] = std::make_pair(true,f.get_frame_metadata((rs2_frame_metadata)i));
else
frame_md.md_attributes[i].first = false;
}

roi_supported = (dev.get() && dev->auto_exposure_enabled);

texture.upload(f);
}

Expand Down Expand Up @@ -882,7 +905,7 @@ class stream_model

void update_ae_roi_rect(const rect& stream_rect, const mouse_info& mouse, std::string& error_message)
{
if (dev.get() && dev->auto_exposure_enabled)
if (roi_checked)
{
// Case 1: Starting Dragging of the ROI rect
// Pre-condition: not capturing already + mouse is down + we are inside stream rect
Expand Down Expand Up @@ -936,7 +959,6 @@ class stream_model
catch (const error& e)
{
error_message = error_to_string(e);
dev->auto_exposure_enabled = false;
}
}
else // If the rect is empty
Expand All @@ -958,7 +980,6 @@ class stream_model
catch (const error& e)
{
error_message = error_to_string(e);
dev->auto_exposure_enabled = false;
}
}
}
Expand Down Expand Up @@ -992,6 +1013,39 @@ class stream_model
update_ae_roi_rect(stream_rect, g, error_message);
}

void show_metadata(const mouse_info& g)
{

auto lines = RS2_FRAME_METADATA_COUNT;
auto flags = ImGuiWindowFlags_ShowBorders;

ImGui::PushStyleColor(ImGuiCol_WindowBg, { 0.3f, 0.3f, 0.3f, 0.5});
ImGui::PushStyleColor(ImGuiCol_TitleBg, { 0.f, 0.25f, 0.3f, 1 });
ImGui::PushStyleColor(ImGuiCol_TitleBgActive, { 0.f, 0.3f, 0.8f, 1 });
ImGui::PushStyleColor(ImGuiCol_Text, { 1, 1, 1, 1 });

std::string label = to_string() << rs2_stream_to_string(stream) << " Stream Metadata #";
ImGui::Begin(label.c_str(), nullptr, flags);

// Print all available frame metadata attributes
for (size_t i=0; i < RS2_FRAME_METADATA_COUNT; i++ )
{
if (frame_md.md_attributes[i].first)
{
label = to_string() << rs2_frame_metadata_to_string((rs2_frame_metadata)i) << " = " << frame_md.md_attributes[i].second;
ImGui::Text("%s", label.c_str());
}
}

ImGui::End();

ImGui::PopStyleColor();
ImGui::PopStyleColor();
ImGui::PopStyleColor();
ImGui::PopStyleColor();

}

float _frame_timeout = 700.0f;
float _min_timeout = 167.0f;
};
Expand Down Expand Up @@ -1068,6 +1122,7 @@ class device_model
std::vector<std::shared_ptr<subdevice_model>> subdevices;
std::map<rs2_stream, stream_model> streams;
bool fullscreen = false;
bool metadata_supported = false;
rs2_stream selected_stream = RS2_STREAM_ANY;

private:
Expand Down Expand Up @@ -1351,7 +1406,6 @@ int main(int, char**) try
auto list = ctx.query_devices(); // Query RealSense connected devices list



// If no device is connected...
while (list.size() == 0)
{
Expand Down Expand Up @@ -1401,6 +1455,7 @@ int main(int, char**) try
data.curr_window = window;
data.mouse = &mouse;


glfwSetWindowUserPointer(window, &data);

glfwSetCursorPosCallback(window, [](GLFWwindow * w, double cx, double cy)
Expand Down Expand Up @@ -1648,8 +1703,7 @@ int main(int, char**) try
if (update_read_only_options)
{
metadata.update_supported(error_message);
if (metadata.supported &&
sub->streaming)
if (metadata.supported && sub->streaming)
{
metadata.update_read_only(error_message);
if (metadata.read_only)
Expand Down Expand Up @@ -1734,7 +1788,7 @@ int main(int, char**) try

auto layout = model.calc_layout(panel_size, 0.f, w - panel_size, (float)h);

for (auto kvp : layout)
for (auto &&kvp : layout)
{
auto&& view_rect = kvp.second;
auto stream = kvp.first;
Expand Down Expand Up @@ -1796,6 +1850,30 @@ int main(int, char**) try
}
}

if (!layout.empty())
{
if (model.streams[stream].roi_supported )
{
ImGui::SameLine((int)ImGui::GetWindowWidth() - 160);
ImGui::Checkbox("[ROI]", &model.streams[stream].roi_checked);

if (ImGui::IsItemHovered())
ImGui::SetTooltip("Auto Exposure Region-of-Interest Selection");
}
else
model.streams[stream].roi_checked = false;
}

// Control metadata overlay widget
ImGui::SameLine((int)ImGui::GetWindowWidth() - 90); // metadata GUI hint
if (!layout.empty())
{
ImGui::Checkbox("[MD]", &model.streams[stream].metadata_displayed);

if (ImGui::IsItemHovered())
ImGui::SetTooltip("Show per-frame metadata");
}

label = to_string() << "Timestamp: " << std::fixed << std::setprecision(3) << model.streams[stream].timestamp
<< ", Domain:";
ImGui::Text("%s", label.c_str());
Expand Down Expand Up @@ -1859,6 +1937,12 @@ int main(int, char**) try
}
}

// Metadata overlay windows shall be drawn after textures to preserve z-buffer functionality
for (auto &&kvp : layout)
{
if (model.streams[kvp.first].metadata_displayed)
model.streams[kvp.first].show_metadata(mouse);
}

ImGui::Render();

Expand Down
2 changes: 2 additions & 0 deletions examples/cpp-data-collect.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,8 @@ void save_data_to_file(list<frame_data> buffer[], string filename = ".\\frames_d
csv << line.str();
}
}

csv.close();
}

int main(int argc, char** argv) try
Expand Down
29 changes: 28 additions & 1 deletion examples/cpp-headless.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
#include <vector>
#include <limits>
#include <iostream>
#include <fstream>

#define STB_IMAGE_WRITE_IMPLEMENTATION
#include "third_party/stb_image_write.h"
Expand All @@ -25,10 +26,32 @@
using namespace rs2;
using namespace std;

void metadata_to_csv(const frame& frm, const std::string& filename)
{
ofstream csv;

csv.open(filename);

cout << "Writing metadata to " << filename << endl;
csv << "Stream," << rs2_stream_to_string(frm.get_stream_type()) << "\nMetadata Attribute,Value\n";

// Record all the available metadata attributes
for (size_t i = 0; i < RS2_FRAME_METADATA_COUNT; i++)
{
if (frm.supports_frame_metadata((rs2_frame_metadata)i))
{
csv << rs2_frame_metadata_to_string((rs2_frame_metadata)i) << ","
<< frm.get_frame_metadata((rs2_frame_metadata)i) << "\n";
}
}

csv.close();
}


int main() try
{
log_to_console(RS2_LOG_SEVERITY_WARN);
//log_to_file(log_severity::debug, "librealsense.log");

context ctx;
auto list = ctx.query_devices();
Expand Down Expand Up @@ -98,6 +121,10 @@ int main() try
bpp,
pixels,
frame.get_width() * bpp );

/* Record per-frame metadata for UVC streams*/
std::string metadata_file_name = rs2::to_string() << "cpp-headless-output-" << stream_type << "-metadata.csv";
metadata_to_csv(frame, metadata_file_name);
}

frames_by_stream.clear();
Expand Down
4 changes: 2 additions & 2 deletions examples/cpp-pointcloud.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ int main(int argc, char * argv[]) try
}

if (frame.get_stream_type() == mapped)
{
{
mapped_tex.upload(frame);
}
}
Expand Down Expand Up @@ -248,7 +248,7 @@ int main(int argc, char * argv[]) try

ImGui::Text("Texture Source:");
ImGui::PushItemWidth(-1);
if (ImGui::Combo(" texture", &selected_stream, stream_names.data(), stream_names.size()))
if (ImGui::Combo(" texture", &selected_stream, stream_names.data(), static_cast<int>(stream_names.size())))
{
stream.stop();
dev.close();
Expand Down
Loading

0 comments on commit 0862cd0

Please sign in to comment.