Label frames with container_id

docs/Build.md

@@ -34,3 +34,12 @@ MkBuildDir Rel
 RelCMake ../
 make -j 4 .
 ```
+
+### Updating libdatadog
+
+Head over to the libdatadog, do your changes and copy the library back to your vendor directory.
+
+```bash
+cp ${workdir}/libdatadog/headers/include/datadog/common.h ${workdir}/ddprof/vendor_gcc_unknown-linux-2.35_Debug/libdatadog-v2.1.0/include/datadog/common.h
+cp ${workdir}/libdatadog/headers/lib/libdatadog_profiling.a ${workdir}/ddprof/vendor_gcc_unknown-linux-2.35_Debug/libdatadog-v2.1.0/lib/
+```

include/container_id.hpp

@@ -0,0 +1,22 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include "container_id_defs.hpp"
+#include "ddprof_defs.hpp"
+#include "ddres_def.hpp"
+
+#include <optional>
+#include <string>
+
+namespace ddprof {
+using ContainerId = std::optional<std::string>;
+// Extract container id information
+// Expects the path to the /proc/<PID>/cgroup file
+DDRes extract_container_id(const std::string &filepath,
+                           ContainerId &container_id);
+
+} // namespace ddprof

include/container_id_defs.hpp

@@ -0,0 +1,17 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include <string_view>
+
+namespace ddprof {
+// default container_id value in case of:
+// - error example when PID is no longer available
+// - we did not lookup (for perf reasons), so this is really unknown
+constexpr static std::string_view k_container_id_unknown = "unknown";
+// no container_id was found within the cgroup file
+constexpr static std::string_view k_container_id_none = "none";
+} // namespace ddprof

include/ddprof_process.hpp

@@ -0,0 +1,63 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include "container_id.hpp"
+#include "ddprof_defs.hpp"
+#include "ddres_def.hpp"
+#include "logger.hpp"
+
+#include <limits>
+#include <sys/types.h>
+#include <unordered_map>
+
+namespace ddprof {
+
+class Process {
+public:
+  explicit Process(pid_t pid) : _pid(pid), _cgroup_ns(kCGroupNsNull) {}
+
+  using CGroupId_t = uint64_t;
+  static constexpr CGroupId_t kCGroupNsNull =
+      std::numeric_limits<CGroupId_t>::max();
+  static constexpr CGroupId_t kCGroupNsError =
+      std::numeric_limits<CGroupId_t>::max() - 1;
+
+  // API only relevant for cgroup v2
+  // lazy read of cgroup id
+  CGroupId_t get_cgroup_ns(std::string_view path_to_proc = "");
+
+  // lazy read of container id
+  const ContainerId &get_container_id(std::string_view path_to_proc = "");
+
+  uint64_t _sample_counter = {};
+
+private:
+  std::string format_cgroup_file(pid_t pid, std::string_view path_to_proc);
+
+  static DDRes read_cgroup_ns(pid_t pid, std::string_view path_to_proc,
+                              CGroupId_t &cgroup);
+
+  pid_t _pid;
+  CGroupId_t _cgroup_ns;
+  ContainerId _container_id;
+};
+
+class ProcessHdr {
+public:
+  ProcessHdr(std::string_view path_to_proc = "")
+      : _path_to_proc(path_to_proc) {}
+  const ContainerId &get_container_id(pid_t pid, bool force = false);
+  void clear(pid_t pid) { _process_map.erase(pid); }
+
+private:
+  constexpr static auto k_nb_samples_container_id_lookup = 100;
+  using ProcessMap = std::unordered_map<pid_t, Process>;
+  ProcessMap _process_map;
+  std::string _path_to_proc = {};
+};
+
+}; // namespace ddprof

include/ddres_list.hpp

@@ -56,6 +56,7 @@
   X(UNITTEST, "unit test error")                                               \
   X(SOCKET, "error during socket operation")                                   \
   X(TEMP_FILE, "error during temporary file creation")                         \
+  X(CGROUP, "error while reading cgroup information")                          \
   X(TSC, "failed to setup TSC")
 
 // generic erno errors available from /usr/include/asm-generic/errno.h

include/unwind_output.hpp

@@ -8,8 +8,10 @@
 #pragma once
 
 #include <stdint.h>
+#include <string>
 #include <vector>
 
+#include "container_id_defs.hpp"
 #include "ddprof_defs.hpp"
 
 typedef struct FunLoc {
@@ -24,8 +26,10 @@ struct UnwindOutput {
   void clear() {
     locs.clear();
     is_incomplete = true;
+    container_id = ddprof::k_container_id_unknown;
   }
   std::vector<FunLoc> locs;
+  std::string container_id;
   int pid;
   int tid;
   bool is_incomplete;

include/unwind_state.hpp

@@ -6,6 +6,7 @@
 #pragma once
 
 #include "ddprof_defs.hpp"
+#include "ddprof_process.hpp"
 #include "ddres_def.hpp"
 #include "dso_hdr.hpp"
 #include "dwfl_hdr.hpp"
@@ -48,6 +49,7 @@ struct UnwindState {
 
   ddprof::DsoHdr dso_hdr;
   SymbolHdr symbol_hdr;
+  ddprof::ProcessHdr process_hdr;
 
   pid_t pid;
   char *stack;

src/container_id.cc

@@ -0,0 +1,62 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#include "container_id.hpp"
+
+#include "ddres.hpp"
+#include "logger.hpp"
+
+#include <fstream>
+#include <regex>
+
+namespace ddprof {
+
+constexpr static std::string_view UUID_SOURCE =
+    "[0-9a-f]{8}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{4}[-_][0-9a-f]{12}";
+constexpr static std::string_view CONTAINER_SOURCE = "[0-9a-f]{64}";
+constexpr static std::string_view TASK_SOURCE = "[0-9a-f]{32}-\\d+";
+
+namespace {
+std::optional<std::string> container_id_from_line(const std::string &line) {
+  std::smatch line_matches;
+  std::smatch container_matches;
+  const std::regex LINE_REGEX(R"(^\d+:[^:]*:(.+)$)");
+  const std::regex CONTAINER_REGEX(
+      "(" + std::string(UUID_SOURCE) + "|" + std::string(CONTAINER_SOURCE) +
+      "|" + std::string(TASK_SOURCE) + ")(?:.scope)? *$");
+
+  if (std::regex_search(line, line_matches, LINE_REGEX)) {
+    std::string match = line_matches[1].str();
+    if (std::regex_search(match, container_matches, CONTAINER_REGEX)) {
+      return container_matches[1].str();
+    }
+  }
+  return std::nullopt;
+}
+
+} // namespace
+
+DDRes extract_container_id(const std::string &filepath,
+                           ContainerId &container_id) {
+  container_id = std::nullopt;
+  std::ifstream cgroup_file(filepath);
+  if (!cgroup_file) {
+    // short lived pids can fail in this case
+    LG_DBG("Failed to open file: %s", filepath.data());
+    return ddres_warn(DD_WHAT_CGROUP);
+  }
+  std::string line;
+  while (std::getline(cgroup_file, line)) {
+    container_id = container_id_from_line(line);
+    if (container_id) {
+      return {};
+    }
+  }
+  // exit path in case we are not within a container
+  container_id = k_container_id_none;
+  return {};
+}
+
+} // namespace ddprof

src/ddprof_process.cc

@@ -0,0 +1,103 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#include "ddprof_process.hpp"
+
+#include "ddres.hpp"
+#include "string_format.hpp"
+
+#include <charconv> // for std::from_chars
+#include <unistd.h>
+#include <vector>
+
+namespace ddprof {
+constexpr auto k_max_buf_cgroup_link = 1024;
+
+std::string Process::format_cgroup_file(pid_t pid,
+                                        std::string_view path_to_proc) {
+  return string_format("%s/proc/%d/cgroup", path_to_proc.data(), pid);
+}
+
+const ContainerId &Process::get_container_id(std::string_view path_to_proc) {
+  if (!_container_id) {
+    extract_container_id(format_cgroup_file(_pid, path_to_proc), _container_id);
+    if (!_container_id) {
+      // file can be gone (short lived pid ?)
+      // store a value to avoid further lookup
+      _container_id = k_container_id_unknown;
+    }
+  }
+  return _container_id;
+}
+
+Process::CGroupId_t Process::get_cgroup_ns(std::string_view path_to_proc) {
+  if (_cgroup_ns == kCGroupNsNull) {
+    read_cgroup_ns(_pid, path_to_proc, _cgroup_ns);
+  }
+  return _cgroup_ns;
+}
+
+DDRes Process::read_cgroup_ns(pid_t pid, std::string_view path_to_proc,
+                              CGroupId_t &cgroup) {
+  cgroup = Process::kCGroupNsError;
+  std::string path =
+      string_format("%s/proc/%d/ns/cgroup", path_to_proc.data(), pid);
+  char buf[k_max_buf_cgroup_link];
+  ssize_t len = readlink(path.c_str(), buf, k_max_buf_cgroup_link - 1);
+  if (len == -1) {
+    // avoid logging as this is frequent
+    return ddres_warn(DD_WHAT_CGROUP);
+  }
+
+  buf[len] = '\0'; // null terminate the string
+
+  std::string_view linkTarget(buf);
+  size_t start = linkTarget.find_last_of('[');
+  size_t end = linkTarget.find_last_of(']');
+
+  if (start == std::string::npos || end == std::string::npos) {
+    DDRES_RETURN_WARN_LOG(DD_WHAT_CGROUP, "Unable to find id %s",
+                          linkTarget.data());
+  }
+
+  std::string_view id_str = linkTarget.substr(start + 1, end - start - 1);
+
+  auto [p, ec] =
+      std::from_chars(id_str.data(), id_str.data() + id_str.size(), cgroup);
+  if (ec == std::errc::invalid_argument ||
+      ec == std::errc::result_out_of_range) {
+    DDRES_RETURN_WARN_LOG(DD_WHAT_CGROUP, "Unable to cgroup to number PID%d",
+                          pid);
+  }
+  return {};
+}
+
+const ContainerId &ProcessHdr::get_container_id(pid_t pid, bool force) {
+  // lookup cgroup
+  static const ContainerId unknown_container_id =
+      ContainerId(k_container_id_unknown);
+  auto it = _process_map.find(pid);
+  if (it == _process_map.end()) {
+    // new process, parse cgroup
+    auto pair = _process_map.try_emplace(pid, pid);
+    if (pair.second) {
+      it = pair.first;
+    } else {
+      LG_WRN("[ProcessHdr] Unable to insert process element");
+      return unknown_container_id;
+    }
+  }
+  // uint64 is big enough that overflow is not a concern
+  // also consequence is just miss-labelling container-id
+  ++(it->second._sample_counter);
+  if (!force &&
+      (it->second._sample_counter) < k_nb_samples_container_id_lookup) {
+    // avoid looking up container_id too often for short-lived pids
+    return unknown_container_id;
+  }
+  return it->second.get_container_id(_path_to_proc);
+}
+
+} // namespace ddprof

src/pprof/ddprof_pprof.cc

@@ -19,7 +19,7 @@
 #include <string.h>
 #include <unistd.h>
 
-#define PPROF_MAX_LABELS 5
+#define PPROF_MAX_LABELS 6
 
 DDRes pprof_create_profile(DDProfPProf *pprof, DDProfContext *ctx) {
   PerfWatcher *watchers = ctx->watchers;
@@ -123,6 +123,12 @@ DDRes pprof_create_profile(DDProfPProf *pprof, DDProfContext *ctx) {
         std::string("include_kernel"),
         include_kernel ? std::string("true") : std::string("false")));
   }
+  {
+    // custom context
+    // Allow the data to be split by container-id
+    pprof->_tags.push_back(std::make_pair(std::string("ddprof.custom_ctx"),
+                                          std::string("container_id")));
+  }
 
   return ddres_init();
 }
@@ -217,6 +223,10 @@ DDRes pprof_aggregate(const UnwindOutput *uw_output,
   char pid_str[sizeof("536870912")] = {}; // reserve space up to 2^29 base-10
   char tid_str[sizeof("536870912")] = {}; // reserve space up to 2^29 base-10
 
+  labels[labels_num].key = to_CharSlice("container_id");
+  labels[labels_num].str = to_CharSlice(uw_output->container_id);
+  ++labels_num;
+
   // Add any configured labels.  Note that TID alone has the same cardinality as
   // (TID;PID) tuples, so except for symbol table overhead it doesn't matter
   // much if TID implies PID for clarity.
@@ -246,6 +256,8 @@ DDRes pprof_aggregate(const UnwindOutput *uw_output,
     }
     ++labels_num;
   }
+  assert(labels_num <= PPROF_MAX_LABELS);
+
   ddog_prof_Sample sample = {
       .locations = {.ptr = locations_buff, .len = cur_loc},
       .values = {.ptr = values, .len = pprof->_nb_values},
@@ -259,7 +271,6 @@ DDRes pprof_aggregate(const UnwindOutput *uw_output,
     DDRES_RETURN_ERROR_LOG(DD_WHAT_PPROF, "Unable to add profile: %s",
                            add_res.err.message.ptr);
   }
-
   return ddres_init();
 }