diff --git a/Cargo.lock b/Cargo.lock index 699cb4d2df9..9f751a14af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -127,6 +127,12 @@ version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25bdb32cbbdce2b519a9cd7df3a678443100e265d5e25ca763b7572a5104f5f3" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "arrayref" version = "0.3.7" @@ -1119,6 +1125,7 @@ dependencies = [ "ciborium", "clap", "criterion-plot", + "csv", "is-terminal", "itertools 0.10.5", "num-traits", @@ -1284,6 +1291,35 @@ dependencies = [ "syn 2.0.71", ] +[[package]] +name = "data-pipeline" +version = "0.0.1" +dependencies = [ + "anyhow", + "arc-swap", + "bytes", + "criterion", + "datadog-ddsketch", + "datadog-trace-normalization", + "datadog-trace-obfuscation", + "datadog-trace-protobuf", + "datadog-trace-utils", + "ddcommon 0.0.1", + "dogstatsd-client", + "either", + "futures", + "httpmock", + "hyper 0.14.28", + "log", + "rand 0.8.5", + "rmp-serde", + "serde", + "serde_json", + "tokio", + "tokio-util", + "uuid", +] + [[package]] name = "datadog-alloc" version = "10.0.0" @@ -1532,6 +1568,7 @@ dependencies = [ "cadence", "chrono", "console-subscriber", + "data-pipeline", "datadog-crashtracker", "datadog-dynamic-configuration", "datadog-ipc", @@ -1624,6 +1661,24 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "datadog-trace-obfuscation" +version = "0.0.1" +dependencies = [ + "anyhow", + "criterion", + "datadog-trace-protobuf", + "datadog-trace-utils", + "ddcommon 0.0.1", + "duplicate", + "log", + "percent-encoding", + "regex", + "serde", + "serde_json", + "url", +] + [[package]] name = "datadog-trace-protobuf" version = "0.0.1" @@ -1960,9 +2015,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "ena" @@ -5647,9 +5702,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.7.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" +checksum = "f8c5f0a0af699448548ad1a2fbf920fb4bee257eae39953ba95cb84891a0446a" dependencies = [ "getrandom", "serde", diff --git a/components-rs/common.h b/components-rs/common.h index 99d9c66a7c9..1044c8ad0a1 100644 --- a/components-rs/common.h +++ b/components-rs/common.h @@ -821,6 +821,8 @@ typedef struct ddog_ContextKey { enum ddog_MetricType _1; } ddog_ContextKey; +typedef struct ddog_AgentInfoReader ddog_AgentInfoReader; + typedef struct ddog_AgentRemoteConfigReader ddog_AgentRemoteConfigReader; typedef struct ddog_AgentRemoteConfigWriter_ShmHandle ddog_AgentRemoteConfigWriter_ShmHandle; diff --git a/components-rs/sidecar.h b/components-rs/sidecar.h index d68d2d4901d..5ed4b738aba 100644 --- a/components-rs/sidecar.h +++ b/components-rs/sidecar.h @@ -306,4 +306,19 @@ void ddog_sidecar_reconnect(struct ddog_SidecarTransport **transport, */ ddog_CharSlice ddog_sidecar_get_crashtracker_unix_socket_path(void); +/** + * Gets an agent info reader. + */ +struct ddog_AgentInfoReader *ddog_get_agent_info_reader(const struct ddog_Endpoint *endpoint); + +/** + * Gets the current agent info environment (or empty if not existing) + */ +ddog_CharSlice ddog_get_agent_info_env(struct ddog_AgentInfoReader *reader, bool *changed); + +/** + * Drops the agent info reader. + */ +void ddog_drop_agent_info_reader(struct ddog_AgentInfoReader*); + #endif /* DDOG_SIDECAR_H */ diff --git a/config.m4 b/config.m4 index 21b6e597a03..74f40fb5f81 100644 --- a/config.m4 +++ b/config.m4 @@ -164,6 +164,7 @@ if test "$PHP_DDTRACE" != "no"; then dnl ddtrace.c comes first, then everything else alphabetically DD_TRACE_PHP_SOURCES="$EXTRA_PHP_SOURCES \ ext/ddtrace.c \ + ext/agent_info.c \ ext/arrays.c \ ext/auto_flush.c \ ext/autoload_php_files.c \ diff --git a/config.w32 b/config.w32 index 9f37c9ef8fd..5c00f88e327 100644 --- a/config.w32 +++ b/config.w32 @@ -15,7 +15,8 @@ if (PHP_DDTRACE != 'no') { var version = PHP_VERSION * 100 + PHP_MINOR_VERSION; - var DDTRACE_EXT_SOURCES = "arrays.c"; + var DDTRACE_EXT_SOURCES = "agent_info.c"; + DDTRACE_EXT_SOURCES += " arrays.c"; DDTRACE_EXT_SOURCES += " auto_flush.c"; DDTRACE_EXT_SOURCES += " autoload_php_files.c"; DDTRACE_EXT_SOURCES += " collect_backtrace.c"; diff --git a/dockerfiles/services/request-replayer/src/index.php b/dockerfiles/services/request-replayer/src/index.php index 5afba6fb99d..9fe8eaff568 100644 --- a/dockerfiles/services/request-replayer/src/index.php +++ b/dockerfiles/services/request-replayer/src/index.php @@ -78,6 +78,7 @@ function decodeDogStatsDMetrics($metrics) define('REQUEST_RC_CONFIGS_FILE', getenv('REQUEST_RC_CONFIGS_FILE') ?: ("$temp_location/rc_configs.json")); define('REQUEST_METRICS_FILE', getenv('REQUEST_METRICS_FILE') ?: ("$temp_location/metrics.json")); define('REQUEST_METRICS_LOG_FILE', getenv('REQUEST_METRICS_LOG_FILE') ?: ("$temp_location/metrics-log.txt")); +define('REQUEST_AGENT_INFO_FILE', getenv('REQUEST_AGENT_INFO_FILE') ?: ("$temp_location/agent-info.txt")); function logRequest($message, $data = '') { @@ -91,8 +92,8 @@ function logRequest($message, $data = '') } set_error_handler(function ($number, $message, $errfile, $errline) { - if (error_reporting() == 0) { - return; + if (!($number & error_reporting())) { + return true; } logRequest("Triggered error $number $message in $errfile on line $errline: " . (new \Exception)->getTraceAsString()); trigger_error($message, $number); @@ -142,6 +143,9 @@ function logRequest($message, $data = '') if (file_exists(REQUEST_NEXT_RESPONSE_FILE)) { unlink(REQUEST_NEXT_RESPONSE_FILE); } + if (file_exists(REQUEST_AGENT_INFO_FILE)) { + unlink(REQUEST_AGENT_INFO_FILE); + } logRequest('Deleted request log'); break; case '/next-response': @@ -214,6 +218,16 @@ function logRequest($message, $data = '') file_put_contents(REQUEST_METRICS_FILE, json_encode($allMetrics)); } break; + case '/set-agent-info': + $raw = file_get_contents('php://input'); + file_put_contents(REQUEST_AGENT_INFO_FILE, $raw); + break; + case '/info': + $file = @file_get_contents(REQUEST_AGENT_INFO_FILE) ?: "{}"; + logRequest('Requested /info endpoint, returning ' . $file); + header("datadog-agent-state: " . sha1($file)); + echo $file; + break; default: $headers = getallheaders(); if (isset($headers['X-Datadog-Diagnostic-Check']) || isset($headers['x-datadog-diagnostic-check'])) { diff --git a/ext/agent_info.c b/ext/agent_info.c new file mode 100644 index 00000000000..a3729c96112 --- /dev/null +++ b/ext/agent_info.c @@ -0,0 +1,22 @@ +#include "agent_info.h" +#include "ddtrace.h" +#include "sidecar.h" +#include "configuration.h" + +ZEND_EXTERN_MODULE_GLOBALS(ddtrace); + +void ddtrace_check_agent_info_env() { + if (DDTRACE_G(agent_info_reader) && ZSTR_LEN(get_DD_ENV()) == 0) { + bool changed; + ddog_CharSlice env = ddog_get_agent_info_env(DDTRACE_G(agent_info_reader), &changed); + if (env.len) { + zend_alter_ini_entry_chars(zai_config_memoized_entries[DDTRACE_CONFIG_DD_ENV].ini_entries[0]->name, env.ptr, env.len, ZEND_INI_USER, ZEND_INI_STAGE_RUNTIME); + } + } +} + +void ddtrace_agent_info_rinit() { + if (ddtrace_endpoint && !ZSTR_LEN(get_global_DD_ENV())) { + DDTRACE_G(agent_info_reader) = ddog_get_agent_info_reader(ddtrace_endpoint); + } +} diff --git a/ext/agent_info.h b/ext/agent_info.h new file mode 100644 index 00000000000..e0c4e10fb16 --- /dev/null +++ b/ext/agent_info.h @@ -0,0 +1,7 @@ +#ifndef DD_AGENT_INFO_H +#define DD_AGENT_INFO_H + +void ddtrace_check_agent_info_env(void); +void ddtrace_agent_info_rinit(void); + +#endif // DD_AGENT_INFO_H diff --git a/ext/ddtrace.c b/ext/ddtrace.c index 0c9bd0771ff..c42688bdf99 100644 --- a/ext/ddtrace.c +++ b/ext/ddtrace.c @@ -99,6 +99,7 @@ #include "ddtrace_arginfo.h" #include "distributed_tracing_headers.h" #include "live_debugger.h" +#include "agent_info.h" #if PHP_VERSION_ID < 70200 #pragma pop_macro("ZVAL_EMPTY_STRING") @@ -655,6 +656,9 @@ static PHP_GSHUTDOWN_FUNCTION(ddtrace) { if (ddtrace_globals->remote_config_state) { ddog_shutdown_remote_config(ddtrace_globals->remote_config_state); } + if (ddtrace_globals->agent_info_reader) { + ddog_drop_agent_info_reader(ddtrace_globals->agent_info_reader); + } zai_hook_gshutdown(); if (ddtrace_globals->telemetry_buffer) { ddog_sidecar_telemetry_buffer_drop(ddtrace_globals->telemetry_buffer); @@ -1497,7 +1501,6 @@ static void dd_rinit_once(void) { static pthread_once_t dd_rinit_once_control = PTHREAD_ONCE_INIT; static void dd_initialize_request(void) { - DDTRACE_G(request_initialized) = true; DDTRACE_G(distributed_trace_id) = (ddtrace_trace_id){0}; DDTRACE_G(distributed_parent_trace_id) = 0; DDTRACE_G(additional_global_tags) = zend_new_array(0); @@ -1507,6 +1510,12 @@ static void dd_initialize_request(void) { zend_hash_init(&DDTRACE_G(propagated_root_span_tags), 8, unused, ZVAL_PTR_DTOR, 0); zend_hash_init(&DDTRACE_G(tracestate_unknown_dd_keys), 8, unused, ZVAL_PTR_DTOR, 0); + // Check for the env first, before the first RC + ddtrace_check_agent_info_env(); + + // Do after env check, so that RC data is not updated before RC init + DDTRACE_G(request_initialized) = true; + ddtrace_sidecar_rinit(); // Things that should only run on the first RINIT after each minit. @@ -1546,6 +1555,8 @@ static void dd_initialize_request(void) { } #endif + ddtrace_agent_info_rinit(); + // Reset compile time after request init hook has compiled ddtrace_compile_time_reset(); @@ -2240,6 +2251,10 @@ void dd_internal_handle_fork(void) { ddog_shutdown_remote_config(DDTRACE_G(remote_config_state)); DDTRACE_G(remote_config_state) = NULL; } + if (DDTRACE_G(agent_info_reader)) { + ddog_drop_agent_info_reader(DDTRACE_G(agent_info_reader)); + DDTRACE_G(agent_info_reader) = NULL; + } ddtrace_seed_prng(); ddtrace_generate_runtime_id(); ddtrace_reset_sidecar_globals(); diff --git a/ext/ddtrace.h b/ext/ddtrace.h index 8f1c14b2f21..481acc43bac 100644 --- a/ext/ddtrace.h +++ b/ext/ddtrace.h @@ -133,6 +133,7 @@ ZEND_BEGIN_MODULE_GLOBALS(ddtrace) ddog_QueueId sidecar_queue_id; ddog_AgentRemoteConfigReader *agent_config_reader; ddog_RemoteConfigState *remote_config_state; + ddog_AgentInfoReader *agent_info_reader; zend_arena *debugger_capture_arena; ddog_Vec_DebuggerPayload exception_debugger_buffer; HashTable active_rc_hooks; diff --git a/ext/priority_sampling/priority_sampling.c b/ext/priority_sampling/priority_sampling.c index b1cb69d6cae..efff1f4800f 100644 --- a/ext/priority_sampling/priority_sampling.c +++ b/ext/priority_sampling/priority_sampling.c @@ -13,6 +13,7 @@ #include "ddtrace.h" #include "span.h" #include "components/log/log.h" +#include "agent_info.h" ZEND_EXTERN_MODULE_GLOBALS(ddtrace); @@ -217,6 +218,9 @@ static void dd_decide_on_sampling(ddtrace_root_span_data *span) { bool explicit_rule = true; if (is_trace_root) { + // when we sample, we need to fetch the env first + ddtrace_check_agent_info_env(); + double default_sample_rate = get_DD_TRACE_SAMPLE_RATE(); sample_rate = default_sample_rate >= 0 ? default_sample_rate : 1; @@ -227,16 +231,22 @@ static void dd_decide_on_sampling(ddtrace_root_span_data *span) { } else { explicit_rule = false; + zval *env = zend_hash_str_find(ddtrace_property_array(&span->property_meta), ZEND_STRL("env")); + if (!env) { + env = &span->property_env; + } + ddtrace_try_read_agent_rate(); + // if we have an empty env... we can default to the cluster env. + if (ZSTR_LEN(get_DD_ENV()) && Z_TYPE_P(env) == IS_STRING && Z_STRLEN_P(env) == 0) { + zval_ptr_dtor(env); + ZVAL_STR_COPY(env, get_DD_ENV()); + } if (DDTRACE_G(agent_rate_by_service)) { - zval *env = zend_hash_str_find(ddtrace_property_array(&span->property_meta), ZEND_STRL("env")); - if (!env) { - env = &span->property_env; - } zval *sample_rate_zv = NULL; zval *service = &span->property_service; - if (Z_TYPE_P(service) == IS_STRING && env && Z_TYPE_P(env) == IS_STRING) { + if (Z_TYPE_P(service) == IS_STRING && Z_TYPE_P(env) == IS_STRING) { zend_string *sample_key = zend_strpprintf(0, "service:%.*s,env:%.*s", (int) Z_STRLEN_P(service), Z_STRVAL_P(service), (int) Z_STRLEN_P(env), Z_STRVAL_P(env)); sample_rate_zv = zend_hash_find(DDTRACE_G(agent_rate_by_service), sample_key); diff --git a/ext/serializer.c b/ext/serializer.c index 173a87dd3be..c6c382d66ab 100644 --- a/ext/serializer.c +++ b/ext/serializer.c @@ -44,6 +44,7 @@ #include "sidecar.h" #include "live_debugger.h" #include "exception_serialize.h" +#include "agent_info.h" ZEND_EXTERN_MODULE_GLOBALS(ddtrace); diff --git a/libdatadog b/libdatadog index 085a91abb8f..a6c810fe0ea 160000 --- a/libdatadog +++ b/libdatadog @@ -1 +1 @@ -Subproject commit 085a91abb8fe6700a6faa528314ada665bc36852 +Subproject commit a6c810fe0eae14ba09d7d61ffe785e511ad589f1 diff --git a/tests/ext/request-replayer/dd_trace_agent_env.phpt b/tests/ext/request-replayer/dd_trace_agent_env.phpt new file mode 100644 index 00000000000..b3a34f86484 --- /dev/null +++ b/tests/ext/request-replayer/dd_trace_agent_env.phpt @@ -0,0 +1,37 @@ +--TEST-- +DDTrace\ExceptionSpanEvent serialization with overridden attributes +--SKIPIF-- + + [ + 'method' => 'PUT', + "header" => [ + "Content-Type: application/json", + "X-Datadog-Test-Session-Token: dd_trace_agent_env", + ], + 'content' => '{"config":{"default_env":"test_env"}}' + ] +]); +file_get_contents("http://request-replayer/set-agent-info", false, $ctx); +?> +--ENV-- +DD_AGENT_HOST=request-replayer +DD_TRACE_AGENT_PORT=80 +DD_TRACE_AGENT_FLUSH_INTERVAL=333 +DD_TRACE_GENERATE_ROOT_SPAN=0 +--INI-- +datadog.env= +datadog.trace.agent_test_session_token=dd_trace_agent_env +--FILE-- +env); + +?> +--EXPECTF-- +string(8) "test_env"