Logger rate limiting

CMakeLists.txt

@@ -218,6 +218,7 @@ set(DD_PROFILING_SOURCES
     src/perf_ringbuffer.cc
     src/perf_watcher.cc
     src/pevent_lib.cc
+    src/ratelimiter.cc
     src/ringbuffer_utils.cc
     src/signal_helper.cc
     src/sys_utils.cc

bench/collatz/CMakeLists.txt

@@ -1,10 +1,6 @@
 # Reuse the same statsd implem as ddprof
-set(STATSD_SRC_FILE ../../src/statsd.cc)
-set(LOGGER_SRC_FILE ../../src/logger.cc)
-set(DDRESLIST_SRC_FILE ../../src/ddres_list.cc)
 set(STATSD_INCLUDE_DIR ../../include)
-
-set(COLLATZ_SRC collatz.cc ${STATSD_SRC_FILE} ${LOGGER_SRC_FILE} ${DDRESLIST_SRC_FILE})
+set(COLLATZ_SRC collatz.cc ../../src/statsd.cc ../../src/logger.cc ../../src/ratelimiter.cc ../../src/ddres_list.cc)
 list(APPEND COLLATZ_DEFINITION_LIST "MYNAME=\"collatz\"" "VER_REV=\"${BUILD_REV}\"")
 
 add_exe(

bench/collatz/collatz.cc

@@ -199,7 +199,7 @@ int main(int c, char **v) {
   int fd_statsd = -1;
   if (const char *path_statsd = getenv("DD_DOGSTATSD_SOCKET");
       path_statsd != nullptr) {
-    statsd_connect(std::string_view(path_statsd), &fd_statsd);
+    ddprof::statsd_connect(std::string_view(path_statsd), &fd_statsd);
   }
 
   // OK, so we want to wait until everyone has started, but if we have more
@@ -223,11 +223,12 @@ int main(int c, char **v) {
       static const char key_stacks[] = "app.collatz.stacks";
       static const char key_funs[] = "app.collatz.functions";
       long val = static_cast<long>(work_end - work_start);
-      statsd_send(fd_statsd, key_ticks, &val, STAT_GAUGE);
-      statsd_send(fd_statsd, key_stacks, &kj, STAT_GAUGE);
+      ddprof::statsd_send(fd_statsd, key_ticks, &val, ddprof::STAT_GAUGE);
+      ddprof::statsd_send(fd_statsd, key_stacks, &kj, ddprof::STAT_GAUGE);
       val = my_counter - last_counter;
-      statsd_send(fd_statsd, key_funs, &val,
-                  STAT_GAUGE); // technically can overflow, but whatever
+      ddprof::statsd_send(
+          fd_statsd, key_funs, &val,
+          ddprof::STAT_GAUGE); // technically can overflow, but whatever
       last_counter = my_counter;
     }
   }

docs/Commands.md

@@ -73,8 +73,8 @@ Advanced settings:
 
 
 Debug options:
-  -l,--log_level TEXT [error]  (Env:DD_PROFILING_NATIVE_LOG_LEVEL)
-                              One of debug, notice, warn, error.
+  -l,--log_level TEXT:{debug,informational,notice,warn,error} [error]  (Env:DD_PROFILING_NATIVE_LOG_LEVEL)
+                              One of debug, informational, notice, warn, error.
   -o,--log_mode TEXT [stdout]  (Env:DD_PROFILING_NATIVE_LOG_MODE)
                               log_level, One of stdout, stderr, syslog, or disabled.
   --show_config [0]           Display the configuration.

include/cap_display.hpp

@@ -7,4 +7,6 @@
 
 #include "ddres_def.hpp"
 
+namespace ddprof {
 DDRes log_capabilities(bool verbose);
+}

include/clocks.hpp

@@ -0,0 +1,46 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0. This product includes software
+// developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present
+// Datadog, Inc.
+
+#pragma once
+
+#include "chrono_utils.hpp"
+
+#include <chrono>
+#include <cstdint>
+#include <time.h>
+
+namespace ddprof {
+
+struct ThreadCpuClock {
+  using duration = std::chrono::nanoseconds;
+  using rep = duration::rep;
+  using period = duration::period;
+  using time_point = std::chrono::time_point<ThreadCpuClock, duration>;
+
+  static constexpr bool is_steady = true;
+
+  static time_point now() noexcept {
+    timespec tp;
+    clock_gettime(CLOCK_THREAD_CPUTIME_ID, &tp);
+    return time_point(timespec_to_duration(tp));
+  }
+};
+
+struct CoarseMonotonicClock {
+  using duration = std::chrono::nanoseconds;
+  using rep = duration::rep;
+  using period = duration::period;
+  using time_point = std::chrono::time_point<ThreadCpuClock, duration>;
+
+  static constexpr bool is_steady = true;
+
+  static time_point now() noexcept {
+    timespec tp;
+    clock_gettime(CLOCK_MONOTONIC_COARSE, &tp);
+    return time_point(timespec_to_duration(tp));
+  }
+};
+
+} // namespace ddprof

include/ddprof_stats.hpp

@@ -9,6 +9,8 @@
 #include "logger.hpp"
 #include "statsd.hpp"
 
+namespace ddprof {
+
 #define X_ENUM(a, b, c) STATS_##a,
 #define STATS_TABLE(X)                                                         \
   X(EVENT_COUNT, "event.count", STAT_GAUGE)                                    \
@@ -68,4 +70,6 @@ DDRes ddprof_stats_get(unsigned int stat, long *out);
 DDRes ddprof_stats_send(std::string_view statsd_socket);
 
 // Print all known stats to the configured log
-void ddprof_stats_print();
+void ddprof_stats_print();
+
+} // namespace ddprof

include/ddres_helpers.hpp

@@ -10,13 +10,16 @@
 #include "logger.hpp"
 
 #include <cstring>
+#include <system_error>
+
+namespace ddprof {
 
 /// Replacement for variadic macro niladic expansion via `__VA_OPT__`, which
 /// is unsupported (boo!) in standards-compliant C static analysis tools and
 /// checkers.
 #define DDRES_NOLOG NULL
 
-/// Standardized way of formating error log
+/// Standardized way of formatting error log
 #define LOG_ERROR_DETAILS(log_func, what)                                      \
   log_func("%s at %s:%u", ddres_error_message(what), __FILE__, __LINE__);
 
@@ -73,7 +76,7 @@
     }                                                                          \
   } while (0)
 
-static inline int ddres_sev_to_log_level(int sev) {
+inline int ddres_sev_to_log_level(int sev) {
   switch (sev) {
   case DD_SEV_ERROR:
     return LL_ERROR;
@@ -118,10 +121,6 @@ static inline int ddres_sev_to_log_level(int sev) {
     }                                                                          \
   } while (0)
 
-// possible improvement : flag to consider warnings as errors
-
-#include <system_error>
-
 /// Evaluate function and return error if -1 (add an error log)
 #define DDRES_CHECK_ERRORCODE(eval, what, ...)                                 \
   do {                                                                         \
@@ -133,3 +132,5 @@ static inline int ddres_sev_to_log_level(int sev) {
       return ddres_error(what);                                                \
     }                                                                          \
   } while (0)
+
+} // namespace ddprof

include/logger.hpp

@@ -5,12 +5,13 @@
 
 #pragma once
 
+#include <chrono>
 #include <stdarg.h>
 
 #include "unlikely.hpp"
 #include "version.hpp"
 
-extern char *LOG_IGNORE;
+namespace ddprof {
 
 enum LOG_OPTS {
   LOG_DISABLE = 0,
@@ -90,11 +91,14 @@ void vlprintfln(int lvl, int fac, const char *name, const char *format,
                 va_list args);
 
 // Setters for global logger context
-bool LOG_setname(const char *name);
+void LOG_setname(const char *name);
 void LOG_setlevel(int lvl);
 int LOG_getlevel();
 void LOG_setfacility(int fac);
 
+void LOG_setratelimit(uint64_t max_log_per_interval,
+                      std::chrono::nanoseconds interval);
+
 /******************************* Logging Macros *******************************/
 #define ABS(__x)                                                               \
   ({                                                                           \
@@ -116,3 +120,5 @@ void LOG_setfacility(int fac);
 #define LG_NFO(...) LG_IF_LVL_OK(LL_INFORMATIONAL, __VA_ARGS__)
 #define LG_DBG(...) LG_IF_LVL_OK(LL_DEBUG, __VA_ARGS__)
 #define PRINT_NFO(...) LG_IF_LVL_OK(-1 * LL_INFORMATIONAL, __VA_ARGS__)
+
+} // namespace ddprof

include/logger_setup.hpp

@@ -5,6 +5,13 @@
 
 #pragma once
 
+#include <cstdint>
+
 namespace ddprof {
-void setup_logger(const char *log_mode, const char *log_level);
-}
+inline constexpr auto kMaxLogPerSecForNonDebug = 100;
+
+void setup_logger(
+    const char *log_mode, const char *log_level,
+    uint64_t max_log_per_sec_for_non_debug = kMaxLogPerSecForNonDebug);
+
+} // namespace ddprof

include/ratelimiter.hpp

@@ -0,0 +1,60 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Taken from facebook/folly
+#pragma once
+
+#include "clocks.hpp"
+
+#include <atomic>
+#include <chrono>
+
+namespace ddprof {
+
+/**
+ * A rate limiter that can rate limit events to N events per M milliseconds.
+ *
+ * It is intended to be fast to check when messages are not being rate limited.
+ * When messages are being rate limited it is slightly slower, as it has to
+ * check the clock each time check() is called in this case.
+ */
+class IntervalRateLimiter {
+public:
+  IntervalRateLimiter(uint64_t max_count_per_interval,
+                      std::chrono::nanoseconds interval) noexcept
+      : _max_count_per_interval(max_count_per_interval), _interval(interval) {}
+
+  bool check() {
+    auto old_count = _count.fetch_add(1, std::memory_order_acq_rel);
+    if (old_count < _max_count_per_interval) {
+      return true;
+    }
+    return check_slow();
+  }
+
+private:
+  bool check_slow() noexcept;
+
+  uint64_t _max_count_per_interval;
+  std::chrono::nanoseconds _interval;
+  // Initialize count_ to the maximum possible value so that the first
+  // call to check() will call checkSlow() to initialize timestamp_,
+  // but subsequent calls will hit the fast-path and avoid checkSlow()
+  std::atomic<uint64_t> _count{std::numeric_limits<uint64_t>::max()};
+  std::atomic<CoarseMonotonicClock::time_point> _interval_end{};
+};
+
+} // namespace ddprof

include/statsd.hpp

@@ -10,6 +10,8 @@
 
 #include "ddres_def.hpp"
 
+namespace ddprof {
+
 enum STAT_TYPES {
   STAT_MS_LONG,
   STAT_MS_FLOAT,
@@ -29,3 +31,5 @@ DDRes statsd_close(int fd);
 
 /* Private */
 DDRes statsd_listen(std::string_view path, int *fd);
+
+} // namespace ddprof

include/user_override.hpp

@@ -6,8 +6,11 @@
 #pragma once
 
 #include "ddres.hpp"
+
 #include <sys/types.h>
 
+namespace ddprof {
+
 struct UIDInfo {
   uid_t uid = -1;
   gid_t gid = -1;
@@ -22,3 +25,5 @@ bool is_root();
 
 // Irreversibly switch to user `user`
 DDRes become_user(const char *user);
+
+} // namespace ddprof

src/cap_display.cc

@@ -11,18 +11,21 @@
 #include <sys/capability.h>
 #include <unistd.h>
 
+namespace ddprof {
+
+namespace {
 struct CapFlag2Text {
   const char *_text;
   cap_flag_t _value;
 };
 
-static const struct CapFlag2Text s_cap_flag_text[] = {
+const struct CapFlag2Text s_cap_flag_text[] = {
     {"CAP_EFFECTIVE", CAP_EFFECTIVE},
     {"CAP_INHERITABLE", CAP_INHERITABLE},
     {"CAP_PERMITTED", CAP_PERMITTED},
 };
 
-enum { SZ_CAP_2_TEXT = 3 };
+} // namespace
 
 DDRes log_capabilities(bool verbose) {
   pid_t const pid = getpid();
@@ -49,3 +52,5 @@ DDRes log_capabilities(bool verbose) {
   cap_free(cap_struct);
   return {};
 }
+
+} // namespace ddprof

src/ddprof_cli.cc

@@ -215,8 +215,10 @@ int DDProfCLI::parse(int argc, const char *argv[]) {
 
   // Debug
   app.add_option("--log_level,-l", log_level,
-                 "One of debug, notice, warn, error.")
+                 "One of debug, informational, notice, warn, error.")
       ->default_val("error")
+      ->check(
+          CLI::IsMember({"debug", "informational", "notice", "warn", "error"}))
       ->group("Debug options")
       ->envname("DD_PROFILING_NATIVE_LOG_LEVEL");
   //

src/ddprof_stats.cc

@@ -9,6 +9,8 @@
 #include <cstring>
 #include <sys/mman.h>
 
+namespace ddprof {
+
 namespace {
 
 // Expand the statsd paths
@@ -159,3 +161,5 @@ void ddprof_stats_print() {
     LG_NTC("%s: %ld", stats_paths[i], ddprof_stats[i]);
   }
 }
+
+} // namespace ddprof