fix(profiling): code provenance using libdatadog exporter [backport-2.12]

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -32,6 +32,8 @@ add_library(dd_wrapper SHARED
     src/crashtracker_interface.cpp
     src/receiver_interface.cpp
     src/synchronized_sample_pool.cpp
+    src/code_provenance.cpp
+    src/code_provenance_interface.cpp
 )
 
 # Add common configuration flags
@@ -96,13 +98,13 @@ set(CRASHTRACKER_EXE_TARGET_NAME ${CRASHTRACKER_EXE_TARGET_NAME} PARENT_SCOPE)
 # will handle the extension artifacts themselves, supplementary files like this one will be left uncopied.
 # One way around this is to propagate the original source dir of the extension, which can be used to deduce the
 # ideal install directory.
-if (INPLACE_LIB_INSTALL_DIR)
+if(INPLACE_LIB_INSTALL_DIR)
     set(LIB_INSTALL_DIR "${INPLACE_LIB_INSTALL_DIR}")
 endif()
 
 # If LIB_INSTALL_DIR is set, install the library.
 # Install one directory up--ddup, crashtracker, and stackv2 are set to the same relative level.
-if (LIB_INSTALL_DIR)
+if(LIB_INSTALL_DIR)
     install(TARGETS dd_wrapper
         LIBRARY DESTINATION ${LIB_INSTALL_DIR}/..
         ARCHIVE DESTINATION ${LIB_INSTALL_DIR}/..
@@ -114,7 +116,7 @@ endif()
 add_cppcheck_target(dd_wrapper
     DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/include
-            ${Datadog_INCLUDE_DIRS}
+    ${Datadog_INCLUDE_DIRS}
     SRC ${CMAKE_CURRENT_SOURCE_DIR}/src
 )
 
@@ -123,7 +125,7 @@ add_infer_target(dd_wrapper)
 add_clangtidy_target(dd_wrapper)
 
 # Add the tests
-if (BUILD_TESTING)
+if(BUILD_TESTING)
     enable_testing()
     add_subdirectory(test)
 endif()

ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <atomic>
+#include <memory>
+#include <mutex>
+#include <optional>
+#include <set>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+namespace Datadog {
+
+struct Package
+{
+    std::string name;
+    std::string version;
+};
+
+static constexpr const char* STDLIB = "stdlib";
+
+class CodeProvenance
+{
+  public:
+    // Public static method to access the CodeProvenance instance
+    static CodeProvenance& get_instance()
+    {
+        static CodeProvenance instance;
+        return instance;
+    }
+
+    // Delete copy constructor and assignment operator to prevent copies
+    CodeProvenance(CodeProvenance const&) = delete;
+    CodeProvenance& operator=(CodeProvenance const&) = delete;
+
+    static void postfork_child();
+
+    void set_enabled(bool enable);
+    bool is_enabled();
+    void set_runtime_version(std::string_view runtime_version);
+    void set_stdlib_path(std::string_view stdlib_path);
+    void add_packages(std::unordered_map<std::string_view, std::string_view> packages);
+    void add_filename(std::string_view filename);
+    std::optional<std::string> try_serialize_to_json_str();
+
+  private:
+    // Mutex to protect the state
+    std::mutex mtx;
+    // Whether this is enabled, set only when DD_PROFILING_ENABLE_CODE_PROVENANCE is set
+    std::atomic_bool enabled{ false };
+    std::string runtime_version;
+    std::string stdlib_path;
+    // Mapping from package name to Package object
+    std::unordered_map<std::string_view, std::unique_ptr<Package>> packages;
+    // Mapping from Package object to list of filenames that are associated with the package
+    std::unordered_map<const Package*, std::set<std::string>> packages_to_files;
+
+    // Private Constructor/Destructor to prevent instantiation/deletion from outside
+    CodeProvenance() = default;
+    ~CodeProvenance() = default;
+
+    void reset();
+    std::string_view get_package_name(std::string_view filename);
+    const Package* add_new_package(std::string_view package_name, std::string_view version);
+};
+}

ddtrace/internal/datadog/profiling/dd_wrapper/include/code_provenance_interface.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string_view>
+#include <unordered_map>
+
+#ifdef __cplusplus
+extern "C"
+{
+#endif
+
+    void code_provenance_enable(bool enable);
+    void code_provenance_set_runtime_version(std::string_view runtime_version);
+    void code_provenance_set_stdlib_path(std::string_view stdlib_path);
+    void code_provenance_add_packages(std::unordered_map<std::string_view, std::string_view> packages);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif

ddtrace/internal/datadog/profiling/dd_wrapper/include/libdatadog_helpers.hpp

@@ -78,6 +78,12 @@ to_slice(std::string_view str)
     return { .ptr = str.data(), .len = str.size() };
 }
 
+inline ddog_ByteSlice
+to_byte_slice(std::string_view str)
+{
+    return { .ptr = reinterpret_cast<const uint8_t*>(str.data()), .len = str.size() };
+}
+
 inline std::string
 err_to_msg(const ddog_Error* err, std::string_view msg)
 {

ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance.cpp

@@ -0,0 +1,177 @@
+#include "code_provenance.hpp"
+
+#include <memory>
+#include <mutex>
+#include <set>
+#include <sstream>
+#include <string>
+#include <string_view>
+#include <unordered_map>
+#include <vector>
+
+namespace Datadog {
+
+void
+Datadog::CodeProvenance::postfork_child()
+{
+    get_instance().mtx.~mutex();            // Destroy the mutex
+    new (&get_instance().mtx) std::mutex(); // Recreate the mutex
+    get_instance().reset();                 // Reset the state
+}
+
+void
+Datadog::CodeProvenance::set_enabled(bool enable)
+{
+    this->enabled.store(enable);
+}
+
+bool
+Datadog::CodeProvenance::is_enabled()
+{
+    return enabled.load();
+}
+
+void
+Datadog::CodeProvenance::set_runtime_version(std::string_view _runtime_version)
+{
+    std::lock_guard<std::mutex> lock(mtx);
+    this->runtime_version = _runtime_version;
+}
+
+void
+Datadog::CodeProvenance::set_stdlib_path(std::string_view _stdlib_path)
+{
+    std::lock_guard<std::mutex> lock(mtx);
+    this->stdlib_path = _stdlib_path;
+}
+
+void
+CodeProvenance::add_packages(std::unordered_map<std::string_view, std::string_view> distributions)
+{
+
+    if (!is_enabled()) {
+        return;
+    }
+
+    std::lock_guard<std::mutex> lock(mtx);
+
+    for (const auto& [package_name, version] : distributions) {
+        auto it = packages.find(package_name);
+        if (it == packages.end()) {
+            add_new_package(package_name, version);
+        }
+    }
+}
+
+void
+CodeProvenance::add_filename(std::string_view filename)
+{
+    if (!is_enabled()) {
+        return;
+    }
+
+    std::string_view package_name = get_package_name(filename);
+    if (package_name.empty() or package_name == STDLIB) {
+        return;
+    }
+
+    std::lock_guard<std::mutex> lock(mtx);
+
+    auto it = packages.find(package_name);
+    if (it == packages.end()) {
+        return;
+    }
+
+    const Package* package = it->second.get();
+    if (package) {
+        if (packages_to_files.find(package) == packages_to_files.end()) {
+            packages_to_files[package] = std::set<std::string>();
+        }
+        packages_to_files[package].insert(std::string(filename));
+    }
+}
+
+std::optional<std::string>
+CodeProvenance::try_serialize_to_json_str()
+{
+    if (!is_enabled()) {
+        return std::nullopt;
+    }
+
+    std::lock_guard<std::mutex> lock(mtx);
+
+    std::ostringstream out;
+    // DEV: Simple JSON serialization, maybe consider using a JSON library.
+    out << "{\"v1\":["; // Start of the JSON array
+    for (const auto& [package, paths] : packages_to_files) {
+        out << "{"; // Start of the JSON object
+        out << "\"name\": \"" << package->name << "\",";
+        out << "\"kind\": \"library\",";
+        out << "\"version\": \"" << package->version << "\",";
+        out << "\"paths\":["; // Start of paths array
+        for (auto it = paths.begin(); it != paths.end(); ++it) {
+            out << "\"" << *it << "\"";
+            if (std::next(it) != paths.end()) {
+                out << ",";
+            }
+        }
+        out << "]";  // End of paths array
+        out << "},"; // End of the JSON object
+    }
+    // Add python runtime information
+    out << "{"; // Start of stdlib JSON object
+    out << "\"name\": \"stdlib\",";
+    out << "\"kind\": \"standard library\",";
+    out << "\"version\": \"" << runtime_version << "\",";
+    out << "\"paths\":[";
+    out << "\"" << stdlib_path << "\"";
+    out << "]";
+    out << "}";  // End of stdlib JSON object
+    out << "]}"; // End of the JSON array
+
+    // Clear the state
+    packages_to_files.clear();
+    return out.str();
+}
+
+void
+CodeProvenance::reset()
+{
+    std::lock_guard<std::mutex> lock(mtx);
+    packages_to_files.clear();
+}
+
+std::string_view
+CodeProvenance::get_package_name(std::string_view filename)
+{
+    // std::regex is slow, so we use a simple string search
+    static const std::string site_packages = "site-packages/";
+
+    size_t start = filename.find(site_packages);
+    if (start == std::string::npos) {
+        return std::string_view(STDLIB);
+    }
+
+    start += site_packages.length();
+    size_t end = filename.find('/', start);
+    if (end == std::string::npos) {
+        return {};
+    }
+
+    return filename.substr(start, end - start);
+}
+
+const Package*
+CodeProvenance::add_new_package(std::string_view package_name, std::string_view version)
+{
+    std::unique_ptr<Package> package = std::make_unique<Package>();
+    package->name = package_name;
+    package->version = version;
+
+    const Package* ret_val = package.get();
+    packages[std::string_view(package->name)] = std::move(package);
+
+    return ret_val;
+}
+
+}

ddtrace/internal/datadog/profiling/dd_wrapper/src/code_provenance_interface.cpp

@@ -0,0 +1,30 @@
+#include "code_provenance_interface.hpp"
+
+#include "code_provenance.hpp"
+
+#include <string_view>
+#include <unordered_map>
+
+void
+code_provenance_enable(bool enable)
+{
+    Datadog::CodeProvenance::get_instance().set_enabled(enable);
+}
+
+void
+code_provenance_set_runtime_version(std::string_view runtime_version)
+{
+    Datadog::CodeProvenance::get_instance().set_runtime_version(runtime_version);
+}
+
+void
+code_provenance_set_stdlib_path(std::string_view stdlib_path)
+{
+    Datadog::CodeProvenance::get_instance().set_stdlib_path(stdlib_path);
+}
+
+void
+code_provenance_add_packages(std::unordered_map<std::string_view, std::string_view> distributions)
+{
+    Datadog::CodeProvenance::get_instance().add_packages(distributions);
+}

ddtrace/internal/datadog/profiling/dd_wrapper/src/ddup_interface.cpp

@@ -1,4 +1,6 @@
 #include "ddup_interface.hpp"
+
+#include "code_provenance.hpp"
 #include "libdatadog_helpers.hpp"
 #include "profile.hpp"
 #include "sample.hpp"
@@ -21,6 +23,7 @@ ddup_postfork_child()
 {
     Datadog::Uploader::postfork_child();
     Datadog::SampleManager::postfork_child();
+    Datadog::CodeProvenance::postfork_child();
 }
 
 void

ddtrace/internal/datadog/profiling/dd_wrapper/src/sample.cpp

@@ -1,5 +1,7 @@
 #include "sample.hpp"
 
+#include "code_provenance.hpp"
+
 #include <algorithm>
 #include <chrono>
 #include <thread>
@@ -29,6 +31,8 @@ Datadog::Sample::push_frame_impl(std::string_view name, std::string_view filenam
     name = profile_state.insert_or_get(name);
     filename = profile_state.insert_or_get(filename);
 
+    CodeProvenance::get_instance().add_filename(filename);
+
     const ddog_prof_Location loc = {
         .mapping = null_mapping, // No support for mappings in Python
         .function = {