From 3c8b52afbd0facff7edd65a35344d7079a320fb7 Mon Sep 17 00:00:00 2001 From: Taegyun Kim Date: Fri, 27 Sep 2024 16:50:56 -0400 Subject: [PATCH] fix(profiling): code provenance using libdatadog exporter (#10796) --- .../profiling/dd_wrapper/CMakeLists.txt | 10 +- .../dd_wrapper/include/code_provenance.hpp | 67 +++++++ .../include/code_provenance_interface.hpp | 18 ++ .../dd_wrapper/include/libdatadog_helpers.hpp | 6 + .../dd_wrapper/src/code_provenance.cpp | 177 ++++++++++++++++++ .../src/code_provenance_interface.cpp | 30 +++ .../dd_wrapper/src/ddup_interface.cpp | 3 + .../profiling/dd_wrapper/src/sample.cpp | 4 + .../profiling/dd_wrapper/src/uploader.cpp | 48 +++-- .../profiling/dd_wrapper/test/CMakeLists.txt | 10 +- .../dd_wrapper/test/code_provenance.cpp | 111 +++++++++++ .../profiling/dd_wrapper/test/test_utils.hpp | 90 ++++++--- .../internal/datadog/profiling/ddup/_ddup.pyi | 1 + .../internal/datadog/profiling/ddup/_ddup.pyx | 39 +++- ddtrace/profiling/profiler.py | 1 + ...ibdd-code-provenance-37f9e3451b0c532c.yaml | 7 + 16 files changed, 569 insertions(+), 53 deletions(-) create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance.hpp create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance_interface.hpp create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance_interface.cpp create mode 100644 ddtrace/internal/datadog/profiling/dd_wrapper/test/code_provenance.cpp create mode 100644 releasenotes/notes/profiling-libdd-code-provenance-37f9e3451b0c532c.yaml diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt index 2a17a0948f5..a344b2e9865 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt @@ -31,6 +31,8 @@ add_library(dd_wrapper SHARED src/crashtracker_interface.cpp src/receiver_interface.cpp src/synchronized_sample_pool.cpp + src/code_provenance.cpp + src/code_provenance_interface.cpp ) # Add common configuration flags @@ -52,13 +54,13 @@ set_target_properties(dd_wrapper PROPERTIES POSITION_INDEPENDENT_CODE ON) # will handle the extension artifacts themselves, supplementary files like this one will be left uncopied. # One way around this is to propagate the original source dir of the extension, which can be used to deduce the # ideal install directory. -if (INPLACE_LIB_INSTALL_DIR) +if(INPLACE_LIB_INSTALL_DIR) set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}") endif() # If LIB_INSTALL_DIR is set, install the library. # Install one directory up--ddup, crashtracker, and stackv2 are set to the same relative level. -if (LIB_INSTALL_DIR) +if(LIB_INSTALL_DIR) install(TARGETS dd_wrapper LIBRARY DESTINATION ${LIB_INSTALL_DIR}/.. ARCHIVE DESTINATION ${LIB_INSTALL_DIR}/.. @@ -70,7 +72,7 @@ endif() add_cppcheck_target(dd_wrapper DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/include - ${Datadog_INCLUDE_DIRS} + ${Datadog_INCLUDE_DIRS} SRC ${CMAKE_CURRENT_SOURCE_DIR}/src ) @@ -79,7 +81,7 @@ add_infer_target(dd_wrapper) add_clangtidy_target(dd_wrapper) # Add the tests -if (BUILD_TESTING) +if(BUILD_TESTING) enable_testing() add_subdirectory(test) endif() diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance.hpp new file mode 100644 index 00000000000..29e0896c418 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance.hpp @@ -0,0 +1,67 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Datadog { + +struct Package +{ + std::string name; + std::string version; +}; + +static constexpr const char* STDLIB = "stdlib"; + +class CodeProvenance +{ + public: + // Public static method to access the CodeProvenance instance + static CodeProvenance& get_instance() + { + static CodeProvenance instance; + return instance; + } + + // Delete copy constructor and assignment operator to prevent copies + CodeProvenance(CodeProvenance const&) = delete; + CodeProvenance& operator=(CodeProvenance const&) = delete; + + static void postfork_child(); + + void set_enabled(bool enable); + bool is_enabled(); + void set_runtime_version(std::string_view runtime_version); + void set_stdlib_path(std::string_view stdlib_path); + void add_packages(std::unordered_map packages); + void add_filename(std::string_view filename); + std::optional try_serialize_to_json_str(); + + private: + // Mutex to protect the state + std::mutex mtx; + // Whether this is enabled, set only when DD_PROFILING_ENABLE_CODE_PROVENANCE is set + std::atomic_bool enabled{ false }; + std::string runtime_version; + std::string stdlib_path; + // Mapping from package name to Package object + std::unordered_map> packages; + // Mapping from Package object to list of filenames that are associated with the package + std::unordered_map> packages_to_files; + + // Private Constructor/Destructor to prevent instantiation/deletion from outside + CodeProvenance() = default; + ~CodeProvenance() = default; + + void reset(); + std::string_view get_package_name(std::string_view filename); + const Package* add_new_package(std::string_view package_name, std::string_view version); +}; +} diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance_interface.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance_interface.hpp new file mode 100644 index 00000000000..71b3c22cd65 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance_interface.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" +{ +#endif + + void code_provenance_enable(bool enable); + void code_provenance_set_runtime_version(std::string_view runtime_version); + void code_provenance_set_stdlib_path(std::string_view stdlib_path); + void code_provenance_add_packages(std::unordered_map packages); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/include/libdatadog_helpers.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/include/libdatadog_helpers.hpp index 3683aa412d9..9952eab8e3f 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/include/libdatadog_helpers.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/include/libdatadog_helpers.hpp @@ -78,6 +78,12 @@ to_slice(std::string_view str) return { .ptr = str.data(), .len = str.size() }; } +inline ddog_ByteSlice +to_byte_slice(std::string_view str) +{ + return { .ptr = reinterpret_cast(str.data()), .len = str.size() }; +} + inline std::string err_to_msg(const ddog_Error* err, std::string_view msg) { diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp new file mode 100644 index 00000000000..0a4a49a4ce5 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp @@ -0,0 +1,177 @@ +#include "code_provenance.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Datadog { + +void +Datadog::CodeProvenance::postfork_child() +{ + get_instance().mtx.~mutex(); // Destroy the mutex + new (&get_instance().mtx) std::mutex(); // Recreate the mutex + get_instance().reset(); // Reset the state +} + +void +Datadog::CodeProvenance::set_enabled(bool enable) +{ + this->enabled.store(enable); +} + +bool +Datadog::CodeProvenance::is_enabled() +{ + return enabled.load(); +} + +void +Datadog::CodeProvenance::set_runtime_version(std::string_view _runtime_version) +{ + std::lock_guard lock(mtx); + this->runtime_version = _runtime_version; +} + +void +Datadog::CodeProvenance::set_stdlib_path(std::string_view _stdlib_path) +{ + std::lock_guard lock(mtx); + this->stdlib_path = _stdlib_path; +} + +void +CodeProvenance::add_packages(std::unordered_map distributions) +{ + + if (!is_enabled()) { + return; + } + + std::lock_guard lock(mtx); + + for (const auto& [package_name, version] : distributions) { + auto it = packages.find(package_name); + if (it == packages.end()) { + add_new_package(package_name, version); + } + } +} + +void +CodeProvenance::add_filename(std::string_view filename) +{ + if (!is_enabled()) { + return; + } + + std::string_view package_name = get_package_name(filename); + if (package_name.empty() or package_name == STDLIB) { + return; + } + + std::lock_guard lock(mtx); + + auto it = packages.find(package_name); + if (it == packages.end()) { + return; + } + + const Package* package = it->second.get(); + if (package) { + if (packages_to_files.find(package) == packages_to_files.end()) { + packages_to_files[package] = std::set(); + } + packages_to_files[package].insert(std::string(filename)); + } +} + +std::optional +CodeProvenance::try_serialize_to_json_str() +{ + if (!is_enabled()) { + return std::nullopt; + } + + std::lock_guard lock(mtx); + + std::ostringstream out; + // DEV: Simple JSON serialization, maybe consider using a JSON library. + out << "{\"v1\":["; // Start of the JSON array + for (const auto& [package, paths] : packages_to_files) { + out << "{"; // Start of the JSON object + out << "\"name\": \"" << package->name << "\","; + out << "\"kind\": \"library\","; + out << "\"version\": \"" << package->version << "\","; + out << "\"paths\":["; // Start of paths array + for (auto it = paths.begin(); it != paths.end(); ++it) { + out << "\"" << *it << "\""; + if (std::next(it) != paths.end()) { + out << ","; + } + } + out << "]"; // End of paths array + out << "},"; // End of the JSON object + } + // Add python runtime information + out << "{"; // Start of stdlib JSON object + out << "\"name\": \"stdlib\","; + out << "\"kind\": \"standard library\","; + out << "\"version\": \"" << runtime_version << "\","; + out << "\"paths\":["; + out << "\"" << stdlib_path << "\""; + out << "]"; + out << "}"; // End of stdlib JSON object + out << "]}"; // End of the JSON array + + // Clear the state + packages_to_files.clear(); + return out.str(); +} + +void +CodeProvenance::reset() +{ + std::lock_guard lock(mtx); + packages_to_files.clear(); +} + +std::string_view +CodeProvenance::get_package_name(std::string_view filename) +{ + // std::regex is slow, so we use a simple string search + static const std::string site_packages = "site-packages/"; + + size_t start = filename.find(site_packages); + if (start == std::string::npos) { + return std::string_view(STDLIB); + } + + start += site_packages.length(); + size_t end = filename.find('/', start); + if (end == std::string::npos) { + return {}; + } + + return filename.substr(start, end - start); +} + +const Package* +CodeProvenance::add_new_package(std::string_view package_name, std::string_view version) +{ + std::unique_ptr package = std::make_unique(); + package->name = package_name; + package->version = version; + + const Package* ret_val = package.get(); + packages[std::string_view(package->name)] = std::move(package); + + return ret_val; +} + +} diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance_interface.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance_interface.cpp new file mode 100644 index 00000000000..8944390cf62 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance_interface.cpp @@ -0,0 +1,30 @@ +#include "code_provenance_interface.hpp" + +#include "code_provenance.hpp" + +#include +#include + +void +code_provenance_enable(bool enable) +{ + Datadog::CodeProvenance::get_instance().set_enabled(enable); +} + +void +code_provenance_set_runtime_version(std::string_view runtime_version) +{ + Datadog::CodeProvenance::get_instance().set_runtime_version(runtime_version); +} + +void +code_provenance_set_stdlib_path(std::string_view stdlib_path) +{ + Datadog::CodeProvenance::get_instance().set_stdlib_path(stdlib_path); +} + +void +code_provenance_add_packages(std::unordered_map distributions) +{ + Datadog::CodeProvenance::get_instance().add_packages(distributions); +} diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp index b49d4605873..255e179f21c 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp @@ -1,4 +1,6 @@ #include "ddup_interface.hpp" + +#include "code_provenance.hpp" #include "libdatadog_helpers.hpp" #include "profile.hpp" #include "sample.hpp" @@ -21,6 +23,7 @@ ddup_postfork_child() { Datadog::Uploader::postfork_child(); Datadog::SampleManager::postfork_child(); + Datadog::CodeProvenance::postfork_child(); } void diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp index 9f939055f9e..f8719a59d1e 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp @@ -1,5 +1,7 @@ #include "sample.hpp" +#include "code_provenance.hpp" + #include #include #include @@ -29,6 +31,8 @@ Datadog::Sample::push_frame_impl(std::string_view name, std::string_view filenam name = profile_state.insert_or_get(name); filename = profile_state.insert_or_get(filename); + CodeProvenance::get_instance().add_filename(filename); + const ddog_prof_Location loc = { .mapping = null_mapping, // No support for mappings in Python .function = { diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp index c09063678b9..3e63b97eb07 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/src/uploader.cpp @@ -1,11 +1,15 @@ #include "uploader.hpp" + +#include "code_provenance.hpp" #include "libdatadog_helpers.hpp" -#include // errno -#include // ofstream +#include // errno +#include // ofstream +#include #include // ostringstream #include // strerror #include // getpid +#include using namespace Datadog; @@ -68,20 +72,32 @@ Datadog::Uploader::upload(ddog_prof_Profile& profile) return ret; } - // Build the request object - const ddog_prof_Exporter_File file = { - .name = to_slice("auto.pprof"), - .file = ddog_Vec_U8_as_slice(&encoded->buffer), - }; - auto build_res = ddog_prof_Exporter_Request_build(ddog_exporter.get(), - encoded->start, - encoded->end, - ddog_prof_Exporter_Slice_File_empty(), - { .ptr = &file, .len = 1 }, - nullptr, - encoded->endpoints_stats, - nullptr, - nullptr); + std::vector files_to_send = { { + .name = to_slice("auto.pprof"), + .file = ddog_Vec_U8_as_slice(&encoded->buffer), + } }; + + // DEV: This function is called with the profile_lock held, and the following + // function call acquires lock on CodeProvenance. + std::optional json_str_opt = CodeProvenance::get_instance().try_serialize_to_json_str(); + if (json_str_opt.has_value() and !json_str_opt.value().empty()) { + files_to_send.push_back({ + .name = to_slice("code-provenance.json"), + .file = to_byte_slice(json_str_opt.value()), + }); + } + + auto build_res = + ddog_prof_Exporter_Request_build(ddog_exporter.get(), + encoded->start, + encoded->end, + ddog_prof_Exporter_Slice_File_empty(), + { .ptr = reinterpret_cast(files_to_send.data()), + .len = static_cast(files_to_send.size()) }, + nullptr, + encoded->endpoints_stats, + nullptr, + nullptr); ddog_prof_EncodedProfile_drop(encoded); if (build_res.tag == diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt index 8b285b9ceca..3be6765c241 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt @@ -1,4 +1,4 @@ -### Testing +# ## Testing FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -10,14 +10,19 @@ FetchContent_MakeAvailable(googletest) include(GoogleTest) include(AnalysisFunc) +FetchContent_Declare(json URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz) +FetchContent_MakeAvailable(json) + function(dd_wrapper_add_test name) add_executable(${name} ${ARGN}) target_include_directories(${name} PRIVATE ../include ) target_link_libraries(${name} PRIVATE + gmock gtest_main dd_wrapper + nlohmann_json::nlohmann_json ) add_ddup_config(${name}) @@ -37,3 +42,6 @@ dd_wrapper_add_test(threading dd_wrapper_add_test(forking forking.cpp ) +dd_wrapper_add_test(code_provenance + code_provenance.cpp +) diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/code_provenance.cpp b/ddtrace/internal/datadog/profiling/dd_wrapper/test/code_provenance.cpp new file mode 100644 index 00000000000..e3ccd338510 --- /dev/null +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/code_provenance.cpp @@ -0,0 +1,111 @@ +#include "code_provenance.hpp" + +#include +#include +#include + +#include + +namespace Datadog { + +using json = nlohmann::json; +using ::testing::UnorderedElementsAreArray; + +class CodeProvenanceTest : public ::testing::Test +{ + protected: + CodeProvenance& cp = CodeProvenance::get_instance(); + void SetUp() override + { + cp.set_enabled(true); + cp.set_runtime_version("3.10.6"); + cp.set_stdlib_path("/usr/lib/python3.10"); + + std::unordered_map packages = { + { "requests", "2.26.0" }, { "urllib3", "1.26.7" }, { "chardet", "4.0.0" }, + { "idna", "3.2" }, { "certifi", "2021.5.30" }, + }; + + cp.add_packages(packages); + } + + void TearDown() override {} +}; + +TEST_F(CodeProvenanceTest, SingletonInstance) +{ + CodeProvenance& cp2 = CodeProvenance::get_instance(); + ASSERT_EQ(&cp, &cp2); +} + +TEST_F(CodeProvenanceTest, SerializeJsonStr) +{ + + cp.add_filename("/usr/lib/python3.10/site-packages/requests/__init__.py"); + cp.add_filename("/usr/lib/python3.10/site-packages/urllib3/__init__.py"); + cp.add_filename("/usr/lib/python3.10/site-packages/chardet/chardet.py"); + cp.add_filename("/usr/lib/python3.10/site-packages/idna/util.py"); + cp.add_filename("/usr/lib/python3.10/site-packages/certifi/cert.py"); + cp.add_filename("/usr/lib/python3.10/site-packages/certifi/__init__.py"); + + std::optional json_str = cp.try_serialize_to_json_str(); + ASSERT_TRUE(json_str.has_value()); + json parsed_json = json::parse(json_str.value()); + + ASSERT_TRUE(parsed_json.contains("v1")); + ASSERT_TRUE(parsed_json["v1"].is_array()); + EXPECT_THAT(parsed_json["v1"], + UnorderedElementsAreArray({ json({ + { "name", "requests" }, + { "kind", "library" }, + { "version", "2.26.0" }, + { "paths", { "/usr/lib/python3.10/site-packages/requests/__init__.py" } }, + }), + json({ + { "name", "urllib3" }, + { "kind", "library" }, + { "version", "1.26.7" }, + { "paths", { "/usr/lib/python3.10/site-packages/urllib3/__init__.py" } }, + }), + json({ + { "name", "chardet" }, + { "kind", "library" }, + { "version", "4.0.0" }, + { "paths", { "/usr/lib/python3.10/site-packages/chardet/chardet.py" } }, + }), + json({ + { "name", "idna" }, + { "kind", "library" }, + { "version", "3.2" }, + { "paths", { "/usr/lib/python3.10/site-packages/idna/util.py" } }, + }), + json({ + { "name", "certifi" }, + { "kind", "library" }, + { "version", "2021.5.30" }, + { "paths", + { + "/usr/lib/python3.10/site-packages/certifi/__init__.py", + "/usr/lib/python3.10/site-packages/certifi/cert.py", + } }, + }), + json({ { "name", "stdlib" }, + { "kind", "standard library" }, + { "version", "3.10.6" }, + { "paths", { "/usr/lib/python3.10" } } }) })); + + json_str = cp.try_serialize_to_json_str(); + parsed_json = json::parse(json_str.value()); + ASSERT_TRUE(parsed_json.contains("v1")); + ASSERT_TRUE(parsed_json["v1"].is_array()); + EXPECT_EQ(parsed_json["v1"].size(), 1); + EXPECT_EQ(parsed_json["v1"][0], + json({ + { "name", "stdlib" }, + { "kind", "standard library" }, + { "version", "3.10.6" }, + { "paths", { "/usr/lib/python3.10" } }, + })); +} + +} // namespace Datadog diff --git a/ddtrace/internal/datadog/profiling/dd_wrapper/test/test_utils.hpp b/ddtrace/internal/datadog/profiling/dd_wrapper/test/test_utils.hpp index b9ea59f857a..f1851f5bc03 100644 --- a/ddtrace/internal/datadog/profiling/dd_wrapper/test/test_utils.hpp +++ b/ddtrace/internal/datadog/profiling/dd_wrapper/test/test_utils.hpp @@ -1,3 +1,4 @@ +#include "code_provenance_interface.hpp" #include "ddup_interface.hpp" #include @@ -8,6 +9,61 @@ #include #include +static constexpr std::array, 3> names = { { + { + "Unreliable", + "Dastardly", + "Careless", + "Clever", + "Inexpensive", + "Righteous", + "Wonderful", + }, + { + "IBM", + "DEC", + "HPE", + "Fujitsu", + "Cray", + "NEC", + "SGI", + }, + { + "Buyer", + "Seller", + "Trader", + "Broker", + "Dealer", + "Merchant", + "Customer", + }, +} }; + +inline static void +configure_code_provenance() +{ + code_provenance_enable(true); + code_provenance_set_runtime_version("3.10.6"); + code_provenance_set_stdlib_path("/usr/lib/python3.10"); + std::unordered_map packages; + + for (size_t i = 0; i < names[0].size(); i++) { + for (size_t j = 0; j < names[1].size(); j++) { + for (size_t k = 0; k < names[2].size(); k++) { + std::string name = + std::string(names[0][i]) + "_" + std::string(names[1][j]) + "_" + std::string(names[2][k]); + packages[name] = "1.0.0"; + } + } + } + + std::unordered_map packages_view; + for (const auto& [key, value] : packages) { + packages_view[key] = value; + } + code_provenance_add_packages(packages_view); +} + inline static void configure(const char* service, const char* env, @@ -26,41 +82,13 @@ configure(const char* service, ddup_config_runtime_version(runtime_version); ddup_config_profiler_version(profiler_version); ddup_config_max_nframes(max_nframes); + configure_code_provenance(); ddup_start(); } inline static std::string get_name() { - constexpr std::array, 3> names = { { - { - "Unreliable", - "Dastardly", - "Careless", - "Clever", - "Inexpensive", - "Righteous", - "Wonderful", - }, - { - "IBM", - "DEC", - "HPE", - "Fujitsu", - "Cray", - "NEC", - "SGI", - }, - { - "Buyer", - "Seller", - "Trader", - "Broker", - "Dealer", - "Merchant", - "Customer", - }, - } }; constexpr auto sz = names[0].size(); thread_local static std::random_device rd; @@ -117,9 +145,9 @@ send_sample(unsigned int id) } for (int i = 0; i < 16; i++) { - std::string file = get_name() + std::to_string(i); + std::string file = "site-packages/" + get_name() + "/file_" + std::to_string(i) + ".py"; std::string func = get_name() + std::to_string(i); - ddup_push_frame(h, file.c_str(), func.c_str(), i, 0); + ddup_push_frame(h, func.c_str(), file.c_str(), i, 0); } ddup_flush_sample(h); ddup_drop_sample(h); diff --git a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi index 47e7e70515e..0df299de338 100644 --- a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi +++ b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyi @@ -13,6 +13,7 @@ def config( url: Optional[str], timeline_enabled: Optional[bool], sample_pool_capacity: Optional[int], + enable_code_provenance: Optional[bool], ) -> None: ... def start() -> None: ... def upload() -> None: ... diff --git a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx index f54d5bad6fd..976605f1ccd 100644 --- a/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx +++ b/ddtrace/internal/datadog/profiling/ddup/_ddup.pyx @@ -1,11 +1,13 @@ # distutils: language = c++ # cython: language_level=3 +import sysconfig from typing import Dict from typing import Optional from typing import Union from libcpp.map cimport map +from libcpp.unordered_map cimport unordered_map from libcpp.utility cimport pair import ddtrace @@ -14,6 +16,7 @@ from .._types import StringType from ..util import ensure_binary_or_empty from ..util import sanitize_string from ddtrace.internal.constants import DEFAULT_SERVICE_NAME +from ddtrace.internal.packages import get_distributions from ddtrace.internal.runtime import get_runtime_id from ddtrace._trace.span import Span @@ -75,6 +78,12 @@ cdef extern from "ddup_interface.hpp": void ddup_flush_sample(Sample *sample) void ddup_drop_sample(Sample *sample) +cdef extern from "code_provenance_interface.hpp": + void code_provenance_enable(bint enable) + void code_provenance_set_runtime_version(string_view runtime_version) + void code_provenance_set_stdlib_path(string_view stdlib_path) + void code_provenance_add_packages(unordered_map[string_view, string_view] packages) + # Create wrappers for cython cdef call_ddup_config_service(bytes service): ddup_config_service(string_view(service, len(service))) @@ -134,7 +143,8 @@ def config( url: StringType = None, timeline_enabled: Optional[bool] = None, output_filename: StringType = None, - sample_pool_capacity: Optional[int] = None) -> None: + sample_pool_capacity: Optional[int] = None, + enable_code_provenance: bool = None) -> None: # Try to provide a ddtrace-specific default service if one is not given service = service or DEFAULT_SERVICE_NAME @@ -167,6 +177,33 @@ def config( if sample_pool_capacity: ddup_config_sample_pool_capacity(clamp_to_uint64_unsigned(sample_pool_capacity)) + # cdef not allowed in if block, so we have to do this here + cdef unordered_map[string_view, string_view] names_and_versions = unordered_map[string_view, string_view]() + # Keep these here to prevent GC from collecting them + dist_names = [] + dist_versions = [] + if enable_code_provenance: + code_provenance_enable(enable_code_provenance) + version_bytes = ensure_binary_or_empty(platform.python_version()) + code_provenance_set_runtime_version( + string_view(version_bytes, len(version_bytes)) + ) + # DEV: Do we also have to pass platsdlib_path, purelib_path, platlib_path? + stdlib_path_bytes = ensure_binary_or_empty(sysconfig.get_path("stdlib")) + code_provenance_set_stdlib_path( + string_view(stdlib_path_bytes, len(stdlib_path_bytes)) + ) + distributions = get_distributions() + for dist in distributions: + dist_name = ensure_binary_or_empty(dist.name) + dist_version = ensure_binary_or_empty(dist.version) + dist_names.append(dist_name) + dist_versions.append(dist_version) + names_and_versions.insert( + pair[string_view, string_view](string_view(dist_name, len(dist_name)), + string_view(dist_version, len(dist_version)))) + code_provenance_add_packages(names_and_versions) + def start() -> None: ddup_start() diff --git a/ddtrace/profiling/profiler.py b/ddtrace/profiling/profiler.py index 90394c48a45..96fa4b7cdfa 100644 --- a/ddtrace/profiling/profiler.py +++ b/ddtrace/profiling/profiler.py @@ -236,6 +236,7 @@ def _build_default_exporters(self): timeline_enabled=config.timeline_enabled, output_filename=config.output_pprof, sample_pool_capacity=config.sample_pool_capacity, + enable_code_provenance=config.code_provenance, ) ddup.start() diff --git a/releasenotes/notes/profiling-libdd-code-provenance-37f9e3451b0c532c.yaml b/releasenotes/notes/profiling-libdd-code-provenance-37f9e3451b0c532c.yaml new file mode 100644 index 00000000000..7377c82b596 --- /dev/null +++ b/releasenotes/notes/profiling-libdd-code-provenance-37f9e3451b0c532c.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + profiling: enables code provenance when using libdatadog exporter, + ``DD_PROFILING_EXPORT_LIBDD_ENABLED``, ``DD_PROFILING_STACK_V2_ENABLED``, + or ``DD_PROFILING_TIMELINE_ENABLED``. +