diff --git a/src/algo/CMakeLists.txt b/src/algo/CMakeLists.txt index 3a130e5ffb..6dafa38bfa 100644 --- a/src/algo/CMakeLists.txt +++ b/src/algo/CMakeLists.txt @@ -2,4 +2,4 @@ # Copyright(c) 2020 Intel Corporation. All Rights Reserved. include(${CMAKE_CURRENT_LIST_DIR}/depth-to-rgb-calibration/CMakeLists.txt) - +include(${CMAKE_CURRENT_LIST_DIR}/thermal-loop/CMakeLists.txt) diff --git a/src/algo/depth-to-rgb-calibration/optimizer.cpp b/src/algo/depth-to-rgb-calibration/optimizer.cpp index 7a0777dca6..09b7ceedbd 100644 --- a/src/algo/depth-to-rgb-calibration/optimizer.cpp +++ b/src/algo/depth-to-rgb-calibration/optimizer.cpp @@ -1557,8 +1557,15 @@ void params::set_depth_resolution( size_t width, size_t height, rs2_ambient_ligh { AC_LOG( DEBUG, " depth resolution= " << width << "x" << height ); // Some parameters are resolution-dependent - bool const XGA = (width == 1024 && height == 768); - bool const VGA = (width == 640 && height == 480); + bool const XGA = ( width == 1024 && height == 768 ); + bool const VGA = ( width == 640 && height == 480 ); + + if( ! XGA && ! VGA ) + { + throw std::runtime_error( to_string() << width << "x" << height + << " this resolution is not supported" ); + } + if( XGA ) { AC_LOG( DEBUG, " changing IR threshold: " << grad_ir_threshold << " -> " << 2.5 << " (because of resolution)" ); @@ -1601,6 +1608,7 @@ void params::set_depth_resolution( size_t width, size_t height, rs2_ambient_ligh } } } + min_weighted_edge_per_section_depth = 50. * ( 480 * 640 ) / ( width * height ); } @@ -1615,6 +1623,7 @@ void params::set_rgb_resolution( size_t width, size_t height ) max_xy_movement_per_calibration[1] = max_xy_movement_per_calibration[2] = 2. * area / hd_area; max_xy_movement_from_origin = 20. * area / hd_area; min_weighted_edge_per_section_rgb = 0.05 * hd_area / area; + } calib const & optimizer::get_calibration() const @@ -1632,19 +1641,7 @@ double optimizer::get_cost() const return _params_curr.cost; } -static -void write_to_file( void const * data, size_t cb, - std::string const & dir, - char const * filename -) -{ - std::string path = dir + filename; - std::fstream f( path, std::ios::out | std::ios::binary ); - if( !f ) - throw std::runtime_error( "failed to open file:\n" + path ); - f.write( (char const *) data, cb ); - f.close(); -} + template< typename T > void write_obj( std::fstream & f, T const & o ) @@ -1652,15 +1649,6 @@ void write_obj( std::fstream & f, T const & o ) f.write( (char const *)&o, sizeof( o ) ); } -template< typename T > -void write_vector_to_file( std::vector< T > const & v, - std::string const & dir, - char const * filename -) -{ - write_to_file( v.data(), v.size() * sizeof( T ), dir, filename ); -} - void write_matlab_camera_params_file( rs2_intrinsics const & _intr_depth, calib const & rgb_calibration, diff --git a/src/algo/depth-to-rgb-calibration/optimizer.h b/src/algo/depth-to-rgb-calibration/optimizer.h index 0390188121..a8b21282e1 100644 --- a/src/algo/depth-to-rgb-calibration/optimizer.h +++ b/src/algo/depth-to-rgb-calibration/optimizer.h @@ -26,6 +26,7 @@ namespace depth_to_rgb_calibration { double step_size = 0; }; + struct params { params(); diff --git a/src/algo/depth-to-rgb-calibration/utils.cpp b/src/algo/depth-to-rgb-calibration/utils.cpp index b77c141bcf..a6f0722112 100644 --- a/src/algo/depth-to-rgb-calibration/utils.cpp +++ b/src/algo/depth-to-rgb-calibration/utils.cpp @@ -10,7 +10,17 @@ namespace librealsense { namespace algo { namespace depth_to_rgb_calibration { - double get_norma(const std::vector& vec) + void write_to_file( void const * data, size_t cb, std::string const & dir, char const * filename ) + { + std::string path = dir + filename; + std::fstream f( path, std::ios::out | std::ios::binary ); + if( ! f ) + throw std::runtime_error( "failed to open file:\n" + path ); + f.write( (char const *)data, cb ); + f.close(); + } + + double get_norma( const std::vector< double3 > & vec ) { double sum = 0; std::for_each( vec.begin(), vec.end(), [&]( double3 const & v ) { sum += v.get_norm(); } ); diff --git a/src/algo/depth-to-rgb-calibration/utils.h b/src/algo/depth-to-rgb-calibration/utils.h index 438515fa78..db550dfe2e 100644 --- a/src/algo/depth-to-rgb-calibration/utils.h +++ b/src/algo/depth-to-rgb-calibration/utils.h @@ -4,7 +4,7 @@ #pragma once #include "calibration-types.h" - +#include namespace librealsense { namespace algo { @@ -32,6 +32,17 @@ namespace depth_to_rgb_calibration { // throw invalid_value_exception if they do. void validate_dsm_params( struct rs2_dsm_params const & dsm_params ); + void + write_to_file( void const * data, size_t cb, std::string const & dir, char const * filename ); + + template < typename T > + void write_vector_to_file( std::vector< T > const & v, + std::string const & dir, + char const * filename ) + { + write_to_file( v.data(), v.size() * sizeof( T ), dir, filename ); + } + } } } diff --git a/src/algo/thermal-loop/CMakeLists.txt b/src/algo/thermal-loop/CMakeLists.txt new file mode 100644 index 0000000000..24f9c09982 --- /dev/null +++ b/src/algo/thermal-loop/CMakeLists.txt @@ -0,0 +1,7 @@ +# License: Apache 2.0. See LICENSE file in root directory. +# Copyright(c) 2020 Intel Corporation. All Rights Reserved. +target_sources(${LRS_TARGET} + PRIVATE + "${CMAKE_CURRENT_LIST_DIR}/l500-thermal-loop.h" + "${CMAKE_CURRENT_LIST_DIR}/l500-thermal-loop.cpp" +) diff --git a/src/algo/thermal-loop/l500-thermal-loop.cpp b/src/algo/thermal-loop/l500-thermal-loop.cpp new file mode 100644 index 0000000000..8e5f87ced8 --- /dev/null +++ b/src/algo/thermal-loop/l500-thermal-loop.cpp @@ -0,0 +1,112 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +#include "l500-thermal-loop.h" +#include "../../l500/l500-private.h" + +namespace librealsense { +namespace algo { +namespace thermal_loop { +namespace l500 { + + +const int thermal_calibration_table::id = 0x317; + + +thermal_calibration_table::thermal_calibration_table( const std::vector< byte > & data, + int resolution ) + : _resolution( resolution ) +{ + float const * header_ptr = (float *)( data.data() + sizeof( ivcam2::table_header ) ); + + auto expected_size = sizeof( ivcam2::table_header ) + sizeof( thermal_table_header ) + + sizeof( thermal_bin ) * resolution; + + if( data.size() != expected_size ) + throw std::runtime_error( librealsense::to_string() + << "data size (" << data.size() + << ") does not meet expected size " << expected_size ); + + _header = *(thermal_table_header *)( header_ptr ); + + auto data_ptr = (thermal_bin *)( data.data() + sizeof( ivcam2::table_header ) + + sizeof( thermal_table_header ) ); + bins.assign( data_ptr, data_ptr + resolution ); +} + + +bool operator==( const thermal_calibration_table & lhs, const thermal_calibration_table & rhs ) +{ + if( lhs.bins.size() != rhs.bins.size() ) + return false; + + if( lhs._header.max_temp != rhs._header.max_temp || lhs._header.min_temp != rhs._header.min_temp + || lhs._header.reference_temp != rhs._header.reference_temp + || lhs._header.valid != rhs._header.valid ) + return false; + + for( auto i = 0; i < rhs.bins.size(); i++ ) + { + if( lhs.bins[i].scale != rhs.bins[i].scale || lhs.bins[i].sheer != rhs.bins[i].sheer + || lhs.bins[i].tx != rhs.bins[i].tx || lhs.bins[i].ty != rhs.bins[i].ty ) + return false; + } + return true; +} + + +double thermal_calibration_table::get_thermal_scale( double hum_temp ) const +{ + auto scale = bins[_resolution - 1].scale; + + auto temp_range = _header.max_temp - _header.min_temp; + // there are 29 bins between min and max temps so 30 equal intervals + auto const interval = temp_range / ( _resolution + 1 ); + // T: |---|---|---| ... |---| + // min 0 1 2 ... 28 max + size_t index = 0; + for( double temp = _header.min_temp; index < _resolution; + ++index, temp += interval ) + { + auto interval_max = temp + interval; + if( hum_temp <= interval_max ) + { + scale = bins[index].scale; + break; + } + } + + // The "scale" is meant to be divided by, but we want something to multiply with! + if( scale == 0 ) + throw std::runtime_error( "invalid 0 scale in thermal table" ); + return 1. / scale; +} + + +std::vector< byte > thermal_calibration_table::build_raw_data() const +{ + std::vector< float > data; + data.resize( sizeof( ivcam2::table_header ) / sizeof( float ), 0 ); + data.push_back( _header.min_temp ); + data.push_back( _header.max_temp ); + data.push_back( _header.reference_temp ); + data.push_back( _header.valid ); + + for( auto i = 0; i < bins.size(); i++ ) + { + data.push_back( bins[i].scale ); + data.push_back( bins[i].sheer ); + data.push_back( bins[i].tx ); + data.push_back( bins[i].ty ); + } + + std::vector< byte > res; + res.assign( (byte *)( data.data() ), (byte *)( data.data() + data.size() ) ); + return res; +} + + +} +} +} +} diff --git a/src/algo/thermal-loop/l500-thermal-loop.h b/src/algo/thermal-loop/l500-thermal-loop.h new file mode 100644 index 0000000000..427274db1c --- /dev/null +++ b/src/algo/thermal-loop/l500-thermal-loop.h @@ -0,0 +1,74 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +#pragma once + +#include +#include "../../types.h" +#include "thermal-calibration-table-interface.h" + +namespace librealsense { +namespace algo { +namespace thermal_loop { +namespace l500 { + + +// RGB_Thermal_Info_CalibInfo table +// ----------------------------------- +// +// The table contains equally spaced bins between min & max temperature. +// +// Each bin has a set of 4 transformation parameters. The transformation maps a point in the RGB +// image in a given temperature to its expected location in the temperature in which the RGB module +// was calibrated. +// +// Reference at: +// https://rsconf.intel.com/display/L500/0x317+RGB+Thermal+Table +// +#pragma pack( push, 1 ) +class thermal_calibration_table : public thermal_calibration_table_interface +{ +public: + static const int id; + + // The header as it's written raw + struct thermal_table_header + { + float min_temp; + float max_temp; + float reference_temp; // not used + float valid; // not used + }; + + // Each bin data, as it's written in the actual raw table + struct thermal_bin + { + float scale; + float sheer; // parameters which affect offset that are not in use + float tx; + float ty; + }; + + // Number of bins *between* min and max: + // bin - size = ( max - min ) / ( resolution + 1 ) + // In the raw calibration table, 29 is implicitly used. + size_t _resolution; + + thermal_table_header _header; + std::vector< thermal_bin > bins; + + thermal_calibration_table() = default; + thermal_calibration_table( const std::vector< byte > & data, int resolution = 29 ); + + double get_thermal_scale( double hum_temp ) const override; + + std::vector< byte > build_raw_data() const override; +}; +#pragma pack( pop ) + +bool operator==( const thermal_calibration_table & lhs, const thermal_calibration_table & rhs ); + +} // namespace l500 +} // namespace thermal_loop +} // namespace algo +} // namespace librealsense diff --git a/src/algo/thermal-loop/thermal-calibration-table-interface.h b/src/algo/thermal-loop/thermal-calibration-table-interface.h new file mode 100644 index 0000000000..b77ef4024c --- /dev/null +++ b/src/algo/thermal-loop/thermal-calibration-table-interface.h @@ -0,0 +1,25 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +#pragma once + +#include + +namespace librealsense { +namespace algo { +namespace thermal_loop { + + +struct thermal_calibration_table_interface +{ + virtual ~thermal_calibration_table_interface() {} + + virtual double get_thermal_scale( double hum_temp ) const = 0; + + virtual std::vector< byte > build_raw_data() const = 0; +}; + + +} // namespace thermal_loop +} // namespace algo +} // namespace librealsense diff --git a/src/depth-to-rgb-calibration.cpp b/src/depth-to-rgb-calibration.cpp index ecd6123ae3..ee58dc8194 100644 --- a/src/depth-to-rgb-calibration.cpp +++ b/src/depth-to-rgb-calibration.cpp @@ -8,10 +8,13 @@ #include "context.h" #include "api.h" // VALIDATE_INTERFACE_NO_THROW #include "algo/depth-to-rgb-calibration/debug.h" - +#include "l500/l500-device.h" +#include "l500/l500-color.h" +#include "algo/depth-to-rgb-calibration/utils.h" using namespace librealsense; namespace impl = librealsense::algo::depth_to_rgb_calibration; +namespace thermal = librealsense::algo::thermal_loop; #define CHECK_IF_NEEDS_TO_STOP() if (_should_continue) _should_continue() @@ -24,14 +27,18 @@ depth_to_rgb_calibration::depth_to_rgb_calibration( std::vector< impl::yuy_t > const & last_yuy_data, impl::algo_calibration_info const & cal_info, impl::algo_calibration_registers const & cal_regs, + rs2_intrinsics yuy_intrinsics, + thermal::thermal_calibration_table_interface const & thermal_table, std::function should_continue ) : _algo( settings ) - , _intr( yuy.get_profile().as< rs2::video_stream_profile >().get_intrinsics() ) + , _raw_intr( yuy_intrinsics ) + , _thermal_intr( _raw_intr ) , _extr(to_raw_extrinsics( depth.get_profile().get_extrinsics_to( yuy.get_profile() ))) , _from( depth.get_profile().get()->profile ) , _to( yuy.get_profile().get()->profile ) , _should_continue( should_continue ) + , _thermal_table( thermal_table ) { AC_LOG( DEBUG, "Setting YUY data" ); auto color_profile = yuy.get_profile().as< rs2::video_stream_profile >(); @@ -41,7 +48,13 @@ depth_to_rgb_calibration::depth_to_rgb_calibration( _last_successful_frame_data = last_yuy_data; // copy -- will be moved to algo else if( ! last_yuy_data.empty() ) AC_LOG( DEBUG, "Not using last successfully-calibrated scene: it's of a different resolution" ); - impl::calib calibration( _intr, _extr ); + + auto scale = thermal_table.get_thermal_scale( settings.hum_temp ); + AC_LOG( DEBUG, " scaling K_rgb by " << scale ); + _thermal_intr.fx = float( _thermal_intr.fx * scale ); + _thermal_intr.fy = float( _thermal_intr.fy * scale ); + + impl::calib calibration( _thermal_intr, _extr ); CHECK_IF_NEEDS_TO_STOP(); @@ -93,6 +106,10 @@ depth_to_rgb_calibration::depth_to_rgb_calibration( void depth_to_rgb_calibration::write_data_to( std::string const & dir ) { _algo.write_data_to( dir ); + + impl::write_to_file( &_raw_intr, sizeof( _raw_intr ), dir, "raw_rgb.intrinsics" ); + + impl::write_vector_to_file( _thermal_table.build_raw_data(), dir, "rgb_thermal_table" ); } @@ -159,8 +176,18 @@ rs2_calibration_status depth_to_rgb_calibration::optimize( // cost= " << params_curr.cost ); AC_LOG( DEBUG, "Optimization successful!" ); - _intr = _algo.get_calibration().get_intrinsics(); - _intr.model = RS2_DISTORTION_INVERSE_BROWN_CONRADY; //restore LRS model + _thermal_intr = _algo.get_calibration().get_intrinsics(); + _thermal_intr.model = RS2_DISTORTION_INVERSE_BROWN_CONRADY; // restore LRS model + + // Override everything in the raw intrinsics except the focal length (fx and fy) + // TODO: AC is not "supposed" to change focal length, but we shouldn't assume this! The + // proper way to do the following is not by restoring the original, but rather by DEscaling + // the results from AC. + auto original_fx = _raw_intr.fx, original_fy = _raw_intr.fy; + _raw_intr = _thermal_intr; + _raw_intr.fx = original_fx; + _raw_intr.fy = original_fy; + _extr = from_raw_extrinsics( _algo.get_calibration().get_extrinsics() ); _dsm_params = _algo.get_dsm_params(); _last_successful_frame_data = _algo.get_yuy_data().orig_frame; // copy -- will be moved to ac_trigger @@ -177,8 +204,9 @@ rs2_calibration_status depth_to_rgb_calibration::optimize( void depth_to_rgb_calibration::debug_calibration( char const * prefix ) { - AC_LOG( DEBUG, AC_F_PREC << " " << prefix << " intr" << _intr ); - AC_LOG( DEBUG, AC_F_PREC << " " << prefix << " extr" << _extr ); - AC_LOG( DEBUG, AC_F_PREC << " " << prefix << " dsm" << _dsm_params ); + AC_LOG( INFO, AC_F_PREC << " " << prefix << " th" << _thermal_intr ); + AC_LOG( INFO, AC_F_PREC << " " << prefix << " raw" << _raw_intr ); + AC_LOG( INFO, AC_F_PREC << " " << prefix << " extr" << _extr ); + AC_LOG( INFO, AC_F_PREC << " " << prefix << " dsm" << _dsm_params ); } diff --git a/src/depth-to-rgb-calibration.h b/src/depth-to-rgb-calibration.h index 843143f80a..cf35a4798a 100644 --- a/src/depth-to-rgb-calibration.h +++ b/src/depth-to-rgb-calibration.h @@ -4,6 +4,7 @@ #pragma once #include "algo/depth-to-rgb-calibration/optimizer.h" +#include "algo/thermal-loop/thermal-calibration-table-interface.h" #include "types.h" #include @@ -23,10 +24,11 @@ namespace librealsense // input/output rs2_extrinsics _extr; - rs2_intrinsics _intr; + rs2_intrinsics _raw_intr; // raw intrinsics for overriding the fw intrinsics + rs2_intrinsics _thermal_intr; // intrinsics with k_thermal for user rs2_dsm_params _dsm_params; std::vector< algo::depth_to_rgb_calibration::yuy_t > _last_successful_frame_data; - + algo::thermal_loop::thermal_calibration_table_interface const & _thermal_table; algo::depth_to_rgb_calibration::optimizer _algo; std::function _should_continue; @@ -40,11 +42,14 @@ namespace librealsense std::vector< algo::depth_to_rgb_calibration::yuy_t > const & last_yuy_data, algo::depth_to_rgb_calibration::algo_calibration_info const & cal_info, algo::depth_to_rgb_calibration::algo_calibration_registers const & cal_regs, + rs2_intrinsics yuy_intrinsics, + algo::thermal_loop::thermal_calibration_table_interface const &, std::function should_continue = nullptr ); rs2_extrinsics const & get_extrinsics() const { return _extr; } - rs2_intrinsics const & get_intrinsics() const { return _intr; } + rs2_intrinsics const & get_raw_intrinsics() const { return _raw_intr; } + rs2_intrinsics const & get_thermal_intrinsics() const { return _thermal_intr; } stream_profile_interface * get_from_profile() const { return _from; } stream_profile_interface * get_to_profile() const { return _to; } rs2_dsm_params const & get_dsm_params() const { return _dsm_params; } diff --git a/src/l500/ac-trigger.cpp b/src/l500/ac-trigger.cpp index 204f9b8328..5ea842c895 100644 --- a/src/l500/ac-trigger.cpp +++ b/src/l500/ac-trigger.cpp @@ -7,8 +7,10 @@ #include "l500-color.h" #include "l500-depth.h" #include "algo/depth-to-rgb-calibration/debug.h" +#include "algo/thermal-loop/l500-thermal-loop.h" #include "log.h" + #ifdef _WIN32 #include #include @@ -927,6 +929,12 @@ namespace ivcam2 { return true; } + rs2_intrinsics read_intrinsics_from_camera( l500_device & dev, + const rs2::stream_profile & profile ) + { + auto vp = profile.as< rs2::video_stream_profile >(); + return dev.get_color_sensor()->get_raw_intrinsics(vp.width(), vp.height() ); + } void ac_trigger::run_algo() { @@ -954,6 +962,7 @@ namespace ivcam2 { // hold up the thread that the frame callbacks are on! float dsm_x_scale, dsm_y_scale, dsm_x_offset, dsm_y_offset; algo::depth_to_rgb_calibration::algo_calibration_info cal_info; + algo::thermal_loop::l500::thermal_calibration_table thermal_table; { auto hwm = _hwm.lock(); if( ! hwm ) @@ -966,6 +975,14 @@ namespace ivcam2 { ivcam2::read_fw_table( *hwm, cal_info.table_id, &cal_info ); + try + { + thermal_table = _dev.get_color_sensor()->get_thermal_table(); + } + catch( ... ) + { + AC_LOG( WARNING, "Could not read thermal_table" ); + } // If the above throw (and they can!) then we catch below and stop... } @@ -995,11 +1012,18 @@ namespace ivcam2 { settings.hum_temp = _temp; settings.ambient = _ambient; settings.receiver_gain = _receiver_gain; - depth_to_rgb_calibration algo( settings, - df, irf, - _cf, _pcf, _last_yuy_data, - cal_info, cal_regs, - should_continue ); + depth_to_rgb_calibration algo( + settings, + df, + irf, + _cf, + _pcf, + _last_yuy_data, + cal_info, + cal_regs, + read_intrinsics_from_camera( _dev, _cf.get_profile() ), + thermal_table, + should_continue ); std::string debug_dir = get_ac_logger().get_active_dir(); if( ! debug_dir.empty() ) @@ -1028,7 +1052,8 @@ namespace ivcam2 { { case RS2_CALIBRATION_SUCCESSFUL: _extr = algo.get_extrinsics(); - _intr = algo.get_intrinsics(); + _raw_intr = algo.get_raw_intrinsics(); + _thermal_intr = algo.get_thermal_intrinsics(); _dsm_params = algo.get_dsm_params(); call_back( status ); // if this throws, we don't want to do the below: _last_temp = _temp; @@ -1211,34 +1236,9 @@ namespace ivcam2 { { auto hwm = _hwm.lock(); if( ! hwm ) - { - AC_LOG( ERROR, "Hardware monitor is inaccessible; cannot read temperature" ); - return 0.; - } - // The temperature may depend on streaming? - std::vector res; + throw std::runtime_error( "HW monitor is inaccessible - stopping algo" ); - try - { - res = hwm->send(command{ TEMPERATURES_GET }); - } - catch (std::exception const & e) - { - AC_LOG(ERROR, - "Failed to get temperatures; hardware monitor in inaccessible: " << e.what()); - return 0.; - } - - if( res.size() < sizeof( temperatures ) ) // New temperatures may get added by FW... - { - AC_LOG( ERROR, - "Failed to get temperatures; result size= " - << res.size() << "; expecting at least " << sizeof( temperatures ) ); - return 0.; - } - auto const & ts = *( reinterpret_cast< temperatures * >( res.data() ) ); - AC_LOG( DEBUG, "HUM temperture is currently " << ts.HUM_temperature << " degrees Celsius" ); - return ts.HUM_temperature; + return _dev.get_color_sensor()->read_temperature(); } @@ -1263,20 +1263,36 @@ namespace ivcam2 { std::string invalid_reason; - // Temperature must be within range or algo may not work right _temp = read_temperature(); - if( _temp < 32. ) + + auto thermal_table_valid = true; + try { - if( ! invalid_reason.empty() ) - invalid_reason += ", "; - invalid_reason += to_string() << "temperature (" << _temp << ") too low (<32)"; + _dev.get_color_sensor()->get_thermal_table(); } - else if( _temp > 46. ) + catch (...) { - if( ! invalid_reason.empty() ) - invalid_reason += ", "; - invalid_reason += to_string() << "temperature (" << _temp << ") too high (>46)"; + thermal_table_valid = false; + } + + if (!thermal_table_valid) + { + if( _temp < 32. ) + { + // If from some reason there is no thermal_table or its not valid + // Temperature must be within range or algo may not work right + if( ! invalid_reason.empty() ) + invalid_reason += ", "; + invalid_reason += to_string() << "temperature (" << _temp << ") too low (<32)"; + } + else if( _temp > 46. ) + { + if( ! invalid_reason.empty() ) + invalid_reason += ", "; + invalid_reason += to_string() << "temperature (" << _temp << ") too high (>46)"; + } } + // Algo was written with specific receiver gain (APD) in mind, depending on // the FW preset (ambient light) diff --git a/src/l500/ac-trigger.h b/src/l500/ac-trigger.h index 4cf7805a62..9cb5090b63 100644 --- a/src/l500/ac-trigger.h +++ b/src/l500/ac-trigger.h @@ -39,7 +39,8 @@ namespace ivcam2 { unsigned _n_cycles = 0; // how many times we've run algo rs2_extrinsics _extr; - rs2_intrinsics _intr; + rs2_intrinsics _raw_intr; + rs2_intrinsics _thermal_intr; rs2_dsm_params _dsm_params; stream_profile_interface* _from_profile = nullptr; stream_profile_interface* _to_profile = nullptr; @@ -172,7 +173,8 @@ namespace ivcam2 { void trigger_calibration( calibration_type type ); rs2_extrinsics const & get_extrinsics() const { return _extr; } - rs2_intrinsics const & get_intrinsics() const { return _intr; } + rs2_intrinsics const & get_raw_intrinsics() const { return _raw_intr; } + rs2_intrinsics const & get_thermal_intrinsics() const { return _thermal_intr; } rs2_dsm_params const & get_dsm_params() const { return _dsm_params; } stream_profile_interface * get_from_profile() const { return _from_profile; } stream_profile_interface * get_to_profile() const { return _to_profile; } diff --git a/src/l500/l500-color.cpp b/src/l500/l500-color.cpp index 5502a93f1d..8aaa30b5ca 100644 --- a/src/l500/l500-color.cpp +++ b/src/l500/l500-color.cpp @@ -12,6 +12,7 @@ #include "proc/color-formats-converter.h" #include "ac-trigger.h" #include "algo/depth-to-rgb-calibration/debug.h" +#include "algo/thermal-loop/l500-thermal-loop.h" namespace librealsense @@ -176,6 +177,24 @@ namespace librealsense environment::get_instance().get_extrinsics_graph().register_extrinsics(*_depth_stream, *_color_stream, _color_extrinsic); register_stream_to_extrinsic_group(*_color_stream, 0); + _thermal_table = [this]() { + hwmon_response response; + auto data = read_fw_table_raw( *_hw_monitor, + algo::thermal_loop::l500::thermal_calibration_table::id, + response ); + if( response != hwm_Success ) + { + AC_LOG( WARNING, + "Failed to read FW table 0x" + << std::hex + << algo::thermal_loop::l500::thermal_calibration_table::id ); + throw invalid_value_exception( + to_string() << "Failed to read FW table 0x" << std::hex + << algo::thermal_loop::l500::thermal_calibration_table::id ); + } + + return algo::thermal_loop::l500::thermal_calibration_table{ data }; + }; _color_device_idx = add_sensor(create_color_device(ctx, color_devs_info)); } @@ -186,7 +205,8 @@ namespace librealsense } - rs2_intrinsics l500_color_sensor::get_intrinsics( const stream_profile& profile ) const + rs2_intrinsics l500_color_sensor::get_raw_intrinsics( uint32_t const width, + uint32_t const height ) const { using namespace ivcam2; @@ -197,7 +217,7 @@ namespace librealsense for( auto i = 0; i < num_of_res; i++ ) { auto model = intrinsic.resolution.intrinsic_resolution[i]; - if( model.height == profile.height && model.width == profile.width ) + if( model.height == height && model.width == width ) { rs2_intrinsics intrinsics; intrinsics.width = model.width; @@ -207,7 +227,9 @@ namespace librealsense intrinsics.ppx = model.ipm.principal_point.x; intrinsics.ppy = model.ipm.principal_point.y; - if( model.distort.radial_k1 || model.distort.radial_k2 || model.distort.tangential_p1 || model.distort.tangential_p2 || model.distort.radial_k3 ) + if( model.distort.radial_k1 || model.distort.radial_k2 + || model.distort.tangential_p1 || model.distort.tangential_p2 + || model.distort.radial_k3 ) { intrinsics.coeffs[0] = model.distort.radial_k1; intrinsics.coeffs[1] = model.distort.radial_k2; @@ -221,7 +243,77 @@ namespace librealsense return intrinsics; } } - throw std::runtime_error( to_string() << "intrinsics for resolution " << profile.width << "," << profile.height << " don't exist" ); + throw std::runtime_error( to_string() << "intrinsics for resolution " << width << "," + << height << " don't exist" ); + } + + double l500_color_sensor::read_temperature() const + { + auto & hwm = *_owner->_hw_monitor; + + std::vector< byte > res; + + try + { + res = hwm.send( command{ TEMPERATURES_GET } ); + } + catch( std::exception const & e ) + { + AC_LOG( ERROR, + "Failed to get temperatures; hardware monitor in inaccessible: " << e.what() ); + return 0.; + } + + if( res.size() < sizeof( temperatures ) ) // New temperatures may get added by FW... + { + AC_LOG( ERROR, + "Failed to get temperatures; result size= " + << res.size() << "; expecting at least " << sizeof( temperatures ) ); + return 0.; + } + auto const & ts = *( reinterpret_cast< temperatures * >( res.data() ) ); + AC_LOG( DEBUG, "HUM temperture is currently " << ts.HUM_temperature << " degrees Celsius" ); + return ts.HUM_temperature; + } + + rs2_intrinsics normalize( const rs2_intrinsics & intr ) + { + auto res = intr; + res.fx = 2 * intr.fx / intr.width; + res.fy = 2 * intr.fy / intr.height; + res.ppx = 2 * intr.ppx / intr.width - 1; + res.ppy = 2 * intr.ppy / intr.height - 1; + + return res; + } + + rs2_intrinsics + denormalize( const rs2_intrinsics & intr, const uint32_t & width, const uint32_t & height ) + { + auto res = intr; + + res.fx = intr.fx * width / 2; + res.fy = intr.fy * height / 2; + res.ppx = ( intr.ppx + 1 ) * width / 2; + res.ppy = ( intr.ppy + 1 ) * height / 2; + + res.width = width; + res.height = height; + + return res; + } + + + rs2_intrinsics l500_color_sensor::get_intrinsics( const stream_profile & profile ) const + { + if( ! _k_thermal_intrinsics) + { + // Until we've calculated temp-based intrinsics, simply use the camera-specified + // intrinsics + return get_raw_intrinsics( profile.width, profile.height ); + } + + return denormalize( *_k_thermal_intrinsics, profile.width, profile.height ); } @@ -236,17 +328,18 @@ namespace librealsense { intr.d[0], intr.d[1], intr.d[2], intr.d[3], intr.d[4] } }; } - void ivcam2::rgb_calibration_table::set_intrinsics( rs2_intrinsics const & i ) { // The table in FW is resolution-agnostic; it can apply to ALL resolutions. To // do this, the focal length and principal point are normalized: + auto norm = normalize( i ); + width = i.width; height = i.height; - intr.fx = 2 * i.fx / i.width; - intr.fy = 2 * i.fy / i.height; - intr.px = 2 * i.ppx / i.width - 1; - intr.py = 2 * i.ppy / i.height - 1; + intr.fx = norm.fx; + intr.fy = norm.fy; + intr.px = norm.ppx; + intr.py = norm.ppy; intr.d[0] = i.coeffs[0]; intr.d[1] = i.coeffs[1]; intr.d[2] = i.coeffs[2]; @@ -280,6 +373,7 @@ namespace librealsense // Intrinsics are resolution-specific, so all the rest of the profile info is not // important _owner->_color_intrinsics_table.reset(); + reset_k_thermal_intrinsics(); } void l500_color_sensor::override_extrinsics( rs2_extrinsics const& extr ) @@ -312,6 +406,22 @@ namespace librealsense throw std::logic_error( "color sensor does not support DSM parameters" ); } + void l500_color_sensor::set_k_thermal_intrinsics( rs2_intrinsics const & intr ) + { + + _k_thermal_intrinsics = std::make_shared< rs2_intrinsics >( normalize(intr) ); + } + + void l500_color_sensor::reset_k_thermal_intrinsics() + { + _k_thermal_intrinsics.reset(); + } + + algo::thermal_loop::l500::thermal_calibration_table l500_color_sensor::get_thermal_table() const + { + return *_owner->_thermal_table; + } + void ivcam2::rgb_calibration_table::update_write_fields() { // We don't touch the version... @@ -343,11 +453,12 @@ namespace librealsense AC_LOG( DEBUG, " done" ); _owner->_color_intrinsics_table.reset(); + reset_k_thermal_intrinsics(); environment::get_instance().get_extrinsics_graph().override_extrinsics( *_owner->_depth_stream, *_owner->_color_stream, - from_raw_extrinsics(table.get_extrinsics())); + from_raw_extrinsics( table.get_extrinsics() ) ); AC_LOG( INFO, "Color sensor calibration has been reset" ); } diff --git a/src/l500/l500-color.h b/src/l500/l500-color.h index 4812426c8c..908717866b 100644 --- a/src/l500/l500-color.h +++ b/src/l500/l500-color.h @@ -11,9 +11,11 @@ #include "stream.h" #include "l500-depth.h" #include "calibrated-sensor.h" +#include "algo/thermal-loop/l500-thermal-loop.h" namespace librealsense { + class l500_color : public virtual l500_device { @@ -39,6 +41,7 @@ namespace librealsense lazy _color_intrinsics_table; lazy> _color_extrinsics_table_raw; std::shared_ptr> _color_extrinsic; + lazy< algo::thermal_loop::l500::thermal_calibration_table > _thermal_table; ivcam2::intrinsic_rgb read_intrinsics_table() const; std::vector get_raw_extrinsics_table() const; @@ -61,15 +64,22 @@ namespace librealsense _state(sensor_state::CLOSED) { } + rs2_intrinsics get_raw_intrinsics( uint32_t width, uint32_t height ) const; + double read_temperature() const; rs2_intrinsics get_intrinsics( const stream_profile& profile ) const override; - + + + // calibrated_sensor void override_intrinsics( rs2_intrinsics const& intr ) override; void override_extrinsics( rs2_extrinsics const& extr ) override; rs2_dsm_params get_dsm_params() const override; void override_dsm_params( rs2_dsm_params const & dsm_params ) override; void reset_calibration() override; + void set_k_thermal_intrinsics( rs2_intrinsics const & intr ); + void reset_k_thermal_intrinsics(); + algo::thermal_loop::l500::thermal_calibration_table get_thermal_table() const; stream_profiles init_stream_profiles() override { @@ -139,6 +149,10 @@ namespace librealsense action_delayer _action_delayer; std::mutex _state_mutex; + // Intrinsics from the the last successful AC( if there was one ) with k - thermal correction, + // We save it normalized such that it can be applied to each resolution + std::shared_ptr< rs2_intrinsics > _k_thermal_intrinsics; + enum class sensor_state { CLOSED, diff --git a/src/l500/l500-device.cpp b/src/l500/l500-device.cpp index 83ad4a2ad0..4503840a15 100644 --- a/src/l500/l500-device.cpp +++ b/src/l500/l500-device.cpp @@ -226,8 +226,9 @@ namespace librealsense get_depth_sensor().override_dsm_params( _autocal->get_dsm_params() ); auto & color_sensor = *get_color_sensor(); - color_sensor.override_intrinsics( _autocal->get_intrinsics() ); + color_sensor.override_intrinsics( _autocal->get_raw_intrinsics() ); color_sensor.override_extrinsics( _autocal->get_extrinsics() ); + color_sensor.set_k_thermal_intrinsics(_autocal->get_thermal_intrinsics()); } notify_of_calibration_change( status ); } ); diff --git a/src/l500/l500-private.h b/src/l500/l500-private.h index ef13436ebb..7253340f18 100644 --- a/src/l500/l500-private.h +++ b/src/l500/l500-private.h @@ -85,6 +85,18 @@ namespace librealsense }; #pragma pack(pop) + static std::vector< byte > + read_fw_table_raw( const hw_monitor & hwm, int table_id, hwmon_response & response ) + { + std::vector< byte > res; + command cmd( fw_cmd::READ_TABLE, table_id ); + auto data = hwm.send( cmd, &response ); + + res.assign( data.data(), data.data() + data.size() ); + + return res; + } + // Read a table from firmware and, if FW says the table is empty, optionally initialize it // using your own code... template< typename T > @@ -93,9 +105,8 @@ namespace librealsense table_header * pheader = nullptr, std::function< void() > init = nullptr ) { - command cmd( fw_cmd::READ_TABLE, table_id ); hwmon_response response; - std::vector data = hwm.send( cmd, &response ); + std::vector< byte > data = read_fw_table_raw( hwm, table_id, response ); size_t expected_size = sizeof( table_header ) + sizeof( T ); switch( response ) { @@ -122,6 +133,7 @@ namespace librealsense default: LOG_DEBUG( "Failed to read FW table 0x" << std::hex << table_id ); + command cmd( fw_cmd::READ_TABLE, table_id ); throw invalid_value_exception( hwmon_error_string( cmd, response ) ); } } diff --git a/unit-tests/algo/d2rgb/compare-scene.h b/unit-tests/algo/d2rgb/compare-scene.h index d6957ead46..80cd871a62 100644 --- a/unit-tests/algo/d2rgb/compare-scene.h +++ b/unit-tests/algo/d2rgb/compare-scene.h @@ -543,7 +543,23 @@ void compare_scene( std::string const & scene_dir, scene_metadata md( scene_dir ); algo::optimizer::settings settings; - read_data_from( bin_dir( scene_dir ) + "settings", &settings ); + read_data_from( join( bin_dir( scene_dir ), "settings" ), &settings ); + + auto scale = 1.; + if (read_thermal_data(scene_dir, + settings.hum_temp, scale)) + { + ci.rgb.fx *= scale; + ci.rgb.fy *= scale; + + auto filename = bin_file( "Kthermal_rgb", 9, 1, "double_00" ) + ".bin"; + TRACE( "Comparing " << filename << " ..." ); + CHECK( compare_to_bin_file( algo::k_matrix( ci.rgb ), scene_dir, filename ) ); + } + else + { + TRACE( "No thermal data found" ); + } algo::optimizer cal( settings, debug_mode ); init_algo( cal, @@ -585,16 +601,17 @@ void compare_scene( std::string const & scene_dir, CHECK( is_scene_valid == matlab_scene_valid ); if( debug_mode ) { - bool spread = read_from< uint8_t >( bin_dir( scene_dir ) + "DirSpread_1x1_uint8_00.bin" ); + bool spread + = read_from< uint8_t >( join( bin_dir( scene_dir ), "DirSpread_1x1_uint8_00.bin" ) ); CHECK( data.edges_dir_spread == spread ); - bool rgbEdgesSpread - = read_from< uint8_t >( bin_dir( scene_dir ) + "rgbEdgesSpread_1x1_uint8_00.bin" ); + bool rgbEdgesSpread = read_from< uint8_t >( + join( bin_dir( scene_dir ), "rgbEdgesSpread_1x1_uint8_00.bin" ) ); CHECK( data.rgb_spatial_spread == rgbEdgesSpread ); - bool depthEdgesSpread - = read_from< uint8_t >( bin_dir( scene_dir ) + "depthEdgesSpread_1x1_uint8_00.bin" ); + bool depthEdgesSpread = read_from< uint8_t >( + join( bin_dir( scene_dir ), "depthEdgesSpread_1x1_uint8_00.bin" ) ); CHECK( data.depth_spatial_spread == depthEdgesSpread ); bool isMovementFromLastSuccess = read_from< uint8_t >( - bin_dir( scene_dir ) + "isMovementFromLastSuccess_1x1_uint8_00.bin" ); + join( bin_dir( scene_dir ), "isMovementFromLastSuccess_1x1_uint8_00.bin" ) ); CHECK( data.is_movement_from_last_success == isMovementFromLastSuccess ); } if( stats ) @@ -1281,7 +1298,7 @@ void compare_scene( std::string const & scene_dir, TRACE( "Comparing " << filename << " ..." ); algo::calib matlab_calib; double matlab_cost = 0; - CHECK( get_calib_from_raw_data( matlab_calib, matlab_cost, scene_dir, filename ) ); + CHECK( get_calib_and_cost_from_raw_data( matlab_calib, matlab_cost, scene_dir, filename ) ); CHECK( compare_calib( new_calibration, cost, matlab_calib, matlab_cost ) ); new_calibration.copy_coefs( matlab_calib ); if( stats ) @@ -1293,7 +1310,7 @@ void compare_scene( std::string const & scene_dir, #if 1 auto vertices = read_vector_from< algo::double3 >( - bin_file( bin_dir( scene_dir ) + "end_vertices", 3, md.n_edges, "double_00.bin" ) ); + bin_file( join( bin_dir( scene_dir ), "end_vertices" ), 3, md.n_edges, "double_00.bin" ) ); if( stats ) { @@ -1313,7 +1330,7 @@ void compare_scene( std::string const & scene_dir, algo::p_matrix p_mat; - auto p_vec = read_vector_from< double >( bin_file( bin_dir( scene_dir ) + "end_p_matrix", + auto p_vec = read_vector_from< double >( bin_file( join( bin_dir( scene_dir ), "end_p_matrix" ), num_of_p_matrix_elements, 1, "double_00.bin" ) ); @@ -1323,7 +1340,7 @@ void compare_scene( std::string const & scene_dir, algo::p_matrix p_mat_opt; auto p_vec_opt - = read_vector_from< double >( bin_file( bin_dir( scene_dir ) + "end_p_matrix_opt", + = read_vector_from< double >( bin_file( join( bin_dir( scene_dir ), "end_p_matrix_opt" ), num_of_p_matrix_elements, 1, "double_00.bin" ) ); @@ -1370,7 +1387,7 @@ void compare_scene( std::string const & scene_dir, // svm - remove xyMovementFromOrigin because its still not implemented auto svm_features_mat = read_vector_from< double >( - bin_file( bin_dir( scene_dir ) + "svm_featuresMat", 10, 1, "double_00.bin" ) ); + bin_file( join( bin_dir( scene_dir ), "svm_featuresMat" ), 10, 1, "double_00.bin" ) ); svm_features_mat.erase( svm_features_mat.begin() + 7 ); auto svm_mat = svm_features; diff --git a/unit-tests/algo/d2rgb/compare-to-bin-file.h b/unit-tests/algo/d2rgb/compare-to-bin-file.h index d4abca02e8..32354ffc83 100644 --- a/unit-tests/algo/d2rgb/compare-to-bin-file.h +++ b/unit-tests/algo/d2rgb/compare-to-bin-file.h @@ -264,7 +264,7 @@ bool compare_to_bin_file( { TRACE( "Comparing " << filename << " ..." ); bool ok = true; - auto bin = read_vector_from< F >(bin_dir(scene_dir) + filename, width, height); + auto bin = read_vector_from< F >( join( bin_dir( scene_dir ), filename ), width, height ); if( bin.size() != size) TRACE( filename << ": {matlab size}" << bin.size() << " != {width}" << width << "x" << height << "{height}" ), ok = false; if( vec.size() != bin.size() ) @@ -315,7 +315,7 @@ bool compare_to_bin_file( } -bool get_calib_from_raw_data( +bool get_calib_and_cost_from_raw_data( algo::calib& calib, double& cost, std::string const & scene_dir, @@ -327,7 +327,7 @@ bool get_calib_from_raw_data( sizeof( algo::matrix_3x3 ) + sizeof( double ); // cost - auto bin = read_vector_from< double >( bin_dir( scene_dir ) + filename ); + auto bin = read_vector_from< double >( join( bin_dir( scene_dir ), filename ) ); if( bin.size() * sizeof( double ) != data_size ) { AC_LOG( DEBUG, "... " << filename << ": {matlab size}" << bin.size() * sizeof(double) << " != " << data_size ); @@ -351,6 +351,7 @@ bool get_calib_from_raw_data( return true; } + bool compare_calib( algo::calib const & calib, double cost, algo::calib calib_from_file, @@ -413,7 +414,28 @@ bool compare_to_bin_file( { TRACE("Comparing " << filename << " ..."); bool ok = true; - auto obj_matlab = read_from< D >(bin_dir(scene_dir) + filename); + auto obj_matlab = read_from< D >( join( bin_dir( scene_dir ), filename ) ); return compare_t(obj_matlab, obj_cpp); +} + +bool read_thermal_data( std::string dir, + double hum_temp, + double & scale ) +{ + try + { + auto vec = read_vector_from< byte >( + join( bin_dir( dir ), "rgb_thermal_table" ) ); + + std::vector thermal_vec( 16, 0 ); // table header + thermal_vec.insert( thermal_vec.end(), vec.begin(), vec.end() ); + thermal::l500::thermal_calibration_table thermal_table( thermal_vec ); + scale = thermal_table.get_thermal_scale( hum_temp ); + return true; + } + catch( std::exception const & ) + { + return false; + } } \ No newline at end of file diff --git a/unit-tests/algo/d2rgb/d2rgb-common.h b/unit-tests/algo/d2rgb/d2rgb-common.h index cf48348618..bae1397572 100644 --- a/unit-tests/algo/d2rgb/d2rgb-common.h +++ b/unit-tests/algo/d2rgb/d2rgb-common.h @@ -10,6 +10,7 @@ #include "../../../src/algo/depth-to-rgb-calibration/utils.h" #include "../../../src/algo/depth-to-rgb-calibration/uvmap.h" +#include "../../../src/algo/thermal-loop/l500-thermal-loop.h" #include "ac-logger.h" #if ! defined( DISABLE_LOG_TO_STDOUT ) @@ -21,6 +22,8 @@ ac_logger LOG_TO_STDOUT; namespace algo = librealsense::algo::depth_to_rgb_calibration; +namespace thermal = librealsense::algo::thermal_loop; + using librealsense::to_string; @@ -42,7 +45,10 @@ void init_algo( algo::optimizer & cal, try { - yuy_last_successful_frame = read_image_file< algo::yuy_t >(dir + yuy_last_successful, camera.rgb.width, camera.rgb.height); + yuy_last_successful_frame + = read_image_file< algo::yuy_t >( join( dir, yuy_last_successful ), + camera.rgb.width, + camera.rgb.height ); } catch (...) { diff --git a/unit-tests/algo/d2rgb/scene-data.h b/unit-tests/algo/d2rgb/scene-data.h index cf4928d758..e4befd9552 100644 --- a/unit-tests/algo/d2rgb/scene-data.h +++ b/unit-tests/algo/d2rgb/scene-data.h @@ -181,14 +181,14 @@ struct scene_metadata scene_metadata( std::string const &scene_dir ) { - std::ifstream(bin_dir(scene_dir) + "yuy_prev_z_i.files") >> rgb_file >> - rgb_prev_file >> rgb_prev_valid_file >> z_file >> ir_file; + std::ifstream( join( bin_dir( scene_dir ), "yuy_prev_z_i.files" ) ) >> rgb_file + >> rgb_prev_file >> z_file >> ir_file >> rgb_prev_valid_file; if( rgb_file.empty() ) throw std::runtime_error( "failed to read file:\n" + bin_dir( scene_dir ) + "yuy_prev_z_i.files" ); if( ir_file.empty() ) throw std::runtime_error( "not enough files in:\n" + bin_dir( scene_dir ) + "yuy_prev_z_i.files" ); - std::string metadata = bin_dir( scene_dir ) + "metadata"; + std::string metadata = join( bin_dir( scene_dir ), "metadata" ); std::fstream f = std::fstream( metadata, std::ios::in | std::ios::binary ); if( !f ) throw std::runtime_error( "failed to read file:\n" + metadata ); @@ -241,7 +241,7 @@ camera_params read_camera_params( std::string const &scene_dir, std::string cons }; params_bin param; - read_data_from( bin_dir( scene_dir ) + filename, ¶m ); + read_data_from( join( bin_dir( scene_dir ), filename ), ¶m ); double coeffs[5] = { 0 }; camera_params ci; @@ -299,7 +299,7 @@ dsm_params read_dsm_params(std::string const &scene_dir, std::string const &file librealsense::algo::depth_to_rgb_calibration::algo_calibration_registers algo_calibration_registers; algo_calibration algo_calib; - std::string dsmparams = bin_dir( scene_dir ) + filename; + std::string dsmparams = join( bin_dir( scene_dir ), filename ); std::fstream f = std::fstream(dsmparams, std::ios::in | std::ios::binary ); if( !f ) throw std::runtime_error( "failed to read file:\n" + dsmparams); diff --git a/unit-tests/algo/d2rgb/test-reproduction.cpp b/unit-tests/algo/d2rgb/test-reproduction.cpp index 225630c02a..7334a99ffe 100644 --- a/unit-tests/algo/d2rgb/test-reproduction.cpp +++ b/unit-tests/algo/d2rgb/test-reproduction.cpp @@ -2,7 +2,7 @@ // Copyright(c) 2020 Intel Corporation. All Rights Reserved. //#cmake:add-file ../../../src/algo/depth-to-rgb-calibration/*.cpp - +//#cmake:add-file ../../../src/algo/thermal-loop/*.cpp // We have our own main #define NO_CATCH_CONFIG_MAIN @@ -129,6 +129,7 @@ class custom_ac_logger : public ac_logger } }; + int main( int argc, char * argv[] ) { custom_ac_logger logger; @@ -203,6 +204,27 @@ int main( int argc, char * argv[] ) settings.receiver_gain = 9; } + // If both the raw intrinsics (pre-thermal) and the thermal table exist, apply a thermal + // manipulation. Otherwise just take the final intrinsics from rgb.calib. + try + { + rs2_intrinsics raw_rgb_intr; + read_binary_file( dir, "raw_rgb.intrinsics", &raw_rgb_intr ); + + auto vec = read_vector_from< byte >( join( dir, "rgb_thermal_table" ) ); + thermal::l500::thermal_calibration_table thermal_table( vec ); + + auto scale = thermal_table.get_thermal_scale( settings.hum_temp ); + AC_LOG( DEBUG, "Thermal {scale}" << scale << " [TH]" ); + raw_rgb_intr.fx = float( raw_rgb_intr.fx * scale ); + raw_rgb_intr.fy = float( raw_rgb_intr.fy * scale ); + camera.rgb = raw_rgb_intr; + } + catch( std::exception const & ) + { + AC_LOG( ERROR, "Could not read raw_rgb.intrinsics or rgb_thermal_table; using rgb.calib [NO-THERMAL]" ); + } + algo::optimizer cal( settings, debug_mode ); std::string status; @@ -263,15 +285,10 @@ int main( int argc, char * argv[] ) TRACE( "\n___\nRESULTS: (" << RS2_API_VERSION_STR << " build 2158)" ); auto intr = cal.get_calibration().get_intrinsics(); + intr.model = RS2_DISTORTION_INVERSE_BROWN_CONRADY; // restore LRS model auto extr = cal.get_calibration().get_extrinsics(); - AC_LOG( DEBUG, AC_D_PREC - << "intr[ " - << intr.width << "x" << intr.height - << " ppx: " << intr.ppx << ", ppy: " << intr.ppy << ", fx: " << intr.fx - << ", fy: " << intr.fy << ", model: " << int( intr.model ) << " coeffs[" - << intr.coeffs[0] << ", " << intr.coeffs[1] << ", " << intr.coeffs[2] - << ", " << intr.coeffs[3] << ", " << intr.coeffs[4] << "] ]" ); - AC_LOG( DEBUG, AC_D_PREC << "extr" << (rs2_extrinsics) extr ); + AC_LOG( DEBUG, AC_D_PREC << "intr" << (rs2_intrinsics)intr ); + AC_LOG( DEBUG, AC_D_PREC << "extr" << (rs2_extrinsics)extr ); AC_LOG( DEBUG, AC_D_PREC << "dsm" << cal.get_dsm_params() ); try @@ -288,6 +305,8 @@ int main( int argc, char * argv[] ) } TRACE( "\n___\nVS:" ); + AC_LOG( DEBUG, AC_D_PREC << "intr" << (rs2_intrinsics)calibration.get_intrinsics() ); + AC_LOG( DEBUG, AC_D_PREC << "extr" << (rs2_extrinsics)calibration.get_extrinsics() ); AC_LOG( DEBUG, AC_D_PREC << "dsm" << camera.dsm_params ); TRACE( "\n___\nSTATUS: " + status ); diff --git a/unit-tests/algo/d2rgb/test-scene-2.cpp b/unit-tests/algo/d2rgb/test-scene-2.cpp index a9e82ba08f..6367d45262 100644 --- a/unit-tests/algo/d2rgb/test-scene-2.cpp +++ b/unit-tests/algo/d2rgb/test-scene-2.cpp @@ -2,7 +2,9 @@ // Copyright(c) 2020 Intel Corporation. All Rights Reserved. //#cmake:add-file ../../../src/algo/depth-to-rgb-calibration/*.cpp +//#cmake:add-file ../../../src/algo/thermal-loop/*.cpp +#define DISABLE_LOG_TO_STDOUT #include "d2rgb-common.h" #include "compare-to-bin-file.h" #include "compare-scene.h" @@ -10,12 +12,14 @@ TEST_CASE("Scene 2", "[d2rgb]") { + ac_logger logger; + // TODO so Travis passes, until we fix the test-case //std::string scene_dir("..\\unit-tests\\algo\\depth-to-rgb-calibration\\19.2.20"); - std::string scene_dir( "C:\\work\\autocal" ); - scene_dir += "\\F9440687\\LongRange_D_768x1024_RGB_1920x1080\\2\\"; + std::string scene_dir( "C:\\work\\autocal\\New\\A\\20_05_2020-Ashrafon-cubic2-4\\ac_4" ); + scene_dir += "\\1\\"; - std::ifstream f( bin_dir( scene_dir ) + "camera_params" ); + std::ifstream f( join( bin_dir( scene_dir ), "camera_params" ) ); if( f.good() ) compare_scene( scene_dir ); else diff --git a/unit-tests/algo/d2rgb/test-scenes.cpp b/unit-tests/algo/d2rgb/test-scenes.cpp index 83efbff47d..f34f613789 100644 --- a/unit-tests/algo/d2rgb/test-scenes.cpp +++ b/unit-tests/algo/d2rgb/test-scenes.cpp @@ -2,6 +2,7 @@ // Copyright(c) 2020 Intel Corporation. All Rights Reserved. //#cmake:add-file ../../../src/algo/depth-to-rgb-calibration/*.cpp +//#cmake:add-file ../../../src/algo/thermal-loop/*.cpp // We have our own main #define NO_CATCH_CONFIG_MAIN diff --git a/unit-tests/algo/thermal-loop/create-synthetic-l500-thermal-table.h b/unit-tests/algo/thermal-loop/create-synthetic-l500-thermal-table.h new file mode 100644 index 0000000000..ba315b4247 --- /dev/null +++ b/unit-tests/algo/thermal-loop/create-synthetic-l500-thermal-table.h @@ -0,0 +1,35 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +#include "../algo-common.h" +#include "algo/thermal-loop/l500-thermal-loop.h" + +namespace thermal = librealsense::algo::thermal_loop::l500; + +// Create a fictitious table with min=0 and max depending on the table size +thermal::thermal_calibration_table +create_synthetic_table( const int table_size = 29, float min_temp = 0, float max_temp = 75 ) +{ + thermal::thermal_calibration_table res; + res._header.min_temp = min_temp; + res._header.max_temp = max_temp; + res._header.reference_temp = 100; // not used + res._header.valid = 1; // not used + res._resolution = table_size; + res.bins.resize( table_size ); + + // [0 - 2.5] --> 0.5 + // (2.5 - 5] --> 1 + // (5 - 7.5] --> 1.5 + // (7.5 - 10] --> 2 + // ... + // (72.5 - 75] --> 15 + for( auto i = 0; i < table_size; i++ ) + { + res.bins[i].scale = ( i + 1 ) * 0.5f; + res.bins[i].sheer = 0; + res.bins[i].tx = 0; + res.bins[i].ty = 0; + } + return res; +} diff --git a/unit-tests/algo/thermal-loop/test-get-scale.cpp b/unit-tests/algo/thermal-loop/test-get-scale.cpp new file mode 100644 index 0000000000..ebf85cabf7 --- /dev/null +++ b/unit-tests/algo/thermal-loop/test-get-scale.cpp @@ -0,0 +1,49 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +//#cmake:add-file ../../../src/algo/thermal-loop/*.cpp +#include "../algo-common.h" +#include "./create-synthetic-l500-thermal-table.h" +#include "algo/thermal-loop/l500-thermal-loop.h" + +using namespace librealsense::algo::thermal_loop::l500; + + + +TEST_CASE("get_scale", "[thermal-loop]") +{ + auto syntetic_table = create_synthetic_table(); + // [0 - 2.5] --> 0.5 + // (2.5 - 5] --> 1 + // (5 - 7.5] --> 1.5 + // (7.5 - 10] --> 2 + // ... + // (72.5 - 75] --> 15 + + std::map< double, double > temp_to_expected_scale = { + { -1, 0.5 }, // under minimum temp + { 0, 0.5 }, + { 2.5, 0.5 }, + { 3.56756, 1 }, + { 5, 1 }, + { 7.5, 1.5 }, + { 7.6, 2 }, + { 73, 14.5 }, + { 75, 14.5 }, // maximum temp + { 78, 14.5 }, // above maximum temp + }; + + for (auto temp_scale : temp_to_expected_scale) + { + TRACE( "checking temp = " << temp_scale.first ); + REQUIRE( syntetic_table.get_thermal_scale( temp_scale.first ) + == 1. / temp_scale.second ); + } +} + +TEST_CASE( "scale_is_zero", "[thermal-loop]" ) +{ + auto syntetic_table = create_synthetic_table(); + syntetic_table.bins[0].scale = 0; + REQUIRE_THROWS( syntetic_table.get_thermal_scale( 0 ) ); +} diff --git a/unit-tests/algo/thermal-loop/test-table-parsing.cpp b/unit-tests/algo/thermal-loop/test-table-parsing.cpp new file mode 100644 index 0000000000..8c154547e8 --- /dev/null +++ b/unit-tests/algo/thermal-loop/test-table-parsing.cpp @@ -0,0 +1,84 @@ +// License: Apache 2.0. See LICENSE file in root directory. +// Copyright(c) 2020 Intel Corporation. All Rights Reserved. + +//#cmake:add-file ../../../src/algo/thermal-loop/*.cpp +#include "../algo-common.h" +#include "./create-synthetic-l500-thermal-table.h" +#include "algo/thermal-loop/l500-thermal-loop.h" + + +using namespace librealsense::algo::thermal_loop::l500; + + +TEST_CASE("parse_thermal_table", "[thermal-loop]") +{ + auto original_table = create_synthetic_table(); + auto raw_data = original_table.build_raw_data(); + thermal_calibration_table table_from_raw( raw_data ); + REQUIRE( original_table == table_from_raw ); +} + +TEST_CASE( "data_size_too_small", "[thermal-loop]" ) +{ + auto syntetic_table = create_synthetic_table(); + auto raw_data = syntetic_table.build_raw_data(); + raw_data.pop_back(); + REQUIRE_THROWS( thermal_calibration_table( raw_data ) ); +} + +TEST_CASE( "data_size_too_large", "[thermal-loop]" ) +{ + auto syntetic_table = create_synthetic_table(); + auto raw_data = syntetic_table.build_raw_data(); + raw_data.push_back( 1 ); + REQUIRE_THROWS( thermal_calibration_table( raw_data ) ); +} + +TEST_CASE( "build_raw_data", "[thermal-loop]" ) +{ + auto syntetic_table = create_synthetic_table(1); + auto raw_data = syntetic_table.build_raw_data(); + + std::vector< byte > raw; + raw.resize( sizeof( thermal_calibration_table::thermal_table_header ), 0 ); + + raw.insert( raw.end(), + (byte *)&( syntetic_table._header.min_temp ), + (byte *)&( syntetic_table._header.min_temp ) + 4 ); + + raw.insert( raw.end(), + (byte *)&( syntetic_table._header.max_temp ), + (byte *)&( syntetic_table._header.max_temp ) + 4 ); + + raw.insert( raw.end(), + (byte *)&( syntetic_table._header.reference_temp ), + (byte *)&( syntetic_table._header.reference_temp ) + 4 ); + + raw.insert( raw.end(), + (byte *)&( syntetic_table._header.valid ), + (byte *)&( syntetic_table._header.valid ) + 4 ); + + for (auto v : syntetic_table.bins) + { + raw.insert( raw.end(), + (byte *)&( v.scale ), + (byte *)&( v.scale ) + 4 ); + + raw.insert( raw.end(), (byte *)&( v.sheer ), (byte *)&( v.sheer ) + 4 ); + raw.insert( raw.end(), (byte *)&( v.tx ), (byte *)&( v.tx ) + 4 ); + raw.insert( raw.end(), (byte *)&( v.ty ), (byte *)&( v.ty ) + 4 ); + } + + CHECK( raw_data == raw ); + REQUIRE_THROWS( thermal_calibration_table( raw_data, 2 ) ); +} + +TEST_CASE( "build_raw_data_no_data", "[thermal-loop]" ) +{ + auto syntetic_table = create_synthetic_table( 0 ); + auto raw_data = syntetic_table.build_raw_data(); + + thermal_calibration_table t( raw_data, 0 ); + + CHECK( t.bins.size() == 0 ); +}