Hunt memory leak

Makefile

@@ -43,6 +43,13 @@ INIT_HOOK_TEST_FILES = $(shell find tests/C2PHP -name '*.phpt' -o -name '*.inc'
 M4_FILES = $(shell find m4 -name '*.m4*' | awk '{ printf "$(BUILD_DIR)/%s\n", $$1 }' ) $(BUILD_DIR)/config.m4
 XDEBUG_SO_FILE = $(shell find $(shell php-config --extension-dir) -type f -name "xdebug*.so" -exec basename {} \; | tail -n 1)
 
+# Make 'sed -i' porable
+ifeq ($(shell uname),Darwin)
+	SED_I = sed -i ''
+else
+	SED_I = sed -i
+endif
+
 all: $(BUILD_DIR)/configure $(SO_FILE)
 
 # The following differentiation exists so we can build only (but always) the relevant files while executing tests
@@ -68,13 +75,13 @@ $(BUILD_DIR)/%Cargo.toml: %Cargo.toml
 	$(Q) echo Copying $*Cargo.toml to $@
 	$(Q) mkdir -p $(dir $@)
 	$(Q) cp -a $*Cargo.toml $@
-	sed -i -E 's/"\.\.\/([^"]*)"/"..\/..\/..\/\1"/' $@
+	$(SED_I) -E 's/"\.\.\/([^"]*)"/"..\/..\/..\/\1"/' $@
 
 $(BUILD_DIR)/Cargo.toml: Cargo.toml
 	$(Q) echo Copying Cargo.toml to $@
 	$(Q) mkdir -p $(dir $@)
 	$(Q) cp -a Cargo.toml $@
-	sed -i -E 's/, "profiling",?//' $@
+	$(SED_I) -E 's/, "profiling",?//' $@
 
 $(BUILD_DIR)/%: %
 	$(Q) echo Copying $* to $@
@@ -87,7 +94,7 @@ JUNIT_RESULTS_DIR := $(shell pwd)
 all: $(BUILD_DIR)/configure $(SO_FILE)
 
 $(BUILD_DIR)/configure: $(M4_FILES) $(BUILD_DIR)/ddtrace.sym
-	$(Q) (cd $(BUILD_DIR); phpize && sed -i 's/\/FAILED/\/\\bFAILED/' $(BUILD_DIR)/run-tests.php) # Fix PHP 5.4 exit code bug when running selected tests (FAILED vs XFAILED)
+	$(Q) (cd $(BUILD_DIR); phpize && $(SED_I) 's/\/FAILED/\/\\bFAILED/' $(BUILD_DIR)/run-tests.php) # Fix PHP 5.4 exit code bug when running selected tests (FAILED vs XFAILED)
 
 $(BUILD_DIR)/Makefile: $(BUILD_DIR)/configure
 	$(Q) (cd $(BUILD_DIR); ./configure --$(if $(RUST_DEBUG_BUILD),enable,disable)-ddtrace-rust-debug)
@@ -290,7 +297,7 @@ test_coverage_collect:
 		--exclude "$(BUILD_DIR)/src/dogstatsd/*" \
 		--exclude "$(BUILD_DIR)/src/dogstatsd/dogstatsd_client/*" \
 		--output-file $(PROJECT_ROOT)/tmp/coverage.info
-	$(Q) sed -i 's+tmp/build_extension/ext+ext+g' $(PROJECT_ROOT)/tmp/coverage.info
+	$(Q) $(SED_I) 's+tmp/build_extension/ext+ext+g' $(PROJECT_ROOT)/tmp/coverage.info
 
 test_coverage_output:
 	$(Q) genhtml \
@@ -1121,6 +1128,7 @@ benchmarks_opcache: benchmarks_run_dependencies call_benchmarks_opcache
 test_opentelemetry_1: global_test_run_dependencies
 	rm -f tests/.scenarios.lock/opentelemetry1/composer.lock
 	$(MAKE) test_scenario_opentelemetry1
+	$(call run_composer_with_retry,tests/Frameworks/Custom/OpenTelemetry,)
 	$(eval TEST_EXTRA_ENV=$(shell [ $(PHP_MAJOR_MINOR) -ge 81 ] && echo "OTEL_PHP_FIBERS_ENABLED=1" || echo '') DD_TRACE_OTEL_ENABLED=1 DD_TRACE_GENERATE_ROOT_SPAN=0)
 	$(call run_tests,--testsuite=opentelemetry1 $(TESTS))
 	$(eval TEST_EXTRA_ENV=)
@@ -1135,7 +1143,9 @@ test_opentracing_beta6: global_test_run_dependencies
 
 test_opentracing_10: global_test_run_dependencies
 	$(MAKE) test_scenario_opentracing10
+	$(call run_composer_with_retry,tests/Frameworks/Custom/OpenTracing,)
 	$(call run_tests,tests/OpenTracer1Unit)
+	$(call run_tests,tests/OpenTracing)
 
 test_integrations: $(TEST_INTEGRATIONS_$(PHP_MAJOR_MINOR))
 test_web: $(TEST_WEB_$(PHP_MAJOR_MINOR))

components-rs/ddtrace.h

@@ -171,6 +171,14 @@ ddog_MaybeError ddog_sidecar_telemetry_buffer_flush(ddog_SidecarTransport **tran
                                                     const ddog_QueueId *queue_id,
                                                     struct ddog_SidecarActionsBuffer *buffer);
 
+void ddog_sidecar_telemetry_register_metric_buffer(struct ddog_SidecarActionsBuffer *buffer,
+                                                   ddog_CharSlice metric_name);
+
+void ddog_sidecar_telemetry_add_span_metric_point_buffer(struct ddog_SidecarActionsBuffer *buffer,
+                                                         ddog_CharSlice metric_name,
+                                                         double metric_value,
+                                                         ddog_CharSlice integration_name);
+
 ddog_MaybeError ddog_sidecar_connect_php(ddog_SidecarTransport **connection,
                                          const char *error_path,
                                          ddog_CharSlice log_level,

components-rs/telemetry.rs

@@ -2,8 +2,10 @@ use datadog_sidecar::interface::blocking::SidecarTransport;
 use datadog_sidecar::interface::{blocking, InstanceId, QueueId, SidecarAction};
 use ddcommon_ffi::slice::AsBytes;
 use ddcommon_ffi::CharSlice;
+use ddcommon::tag::Tag;
 use ddtelemetry::data;
 use ddtelemetry::data::{Dependency, Integration};
+use ddtelemetry::metrics::MetricContext;
 use ddtelemetry::worker::TelemetryActions;
 use ddtelemetry_ffi::{try_c, MaybeError};
 use std::error::Error;
@@ -127,3 +129,37 @@ pub extern "C" fn ddog_sidecar_telemetry_buffer_flush(
 
     MaybeError::None
 }
+
+#[no_mangle]
+pub unsafe extern "C" fn ddog_sidecar_telemetry_register_metric_buffer(
+    buffer: &mut SidecarActionsBuffer,
+    metric_name: CharSlice,
+) {
+
+    buffer.buffer.push(SidecarAction::RegisterTelemetryMetric(MetricContext {
+        name: metric_name.to_utf8_lossy().into_owned(),
+        namespace: data::metrics::MetricNamespace::Tracers,
+        metric_type: data::metrics::MetricType::Count,
+        tags: Vec::default(),
+        common: false,
+    }));
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn ddog_sidecar_telemetry_add_span_metric_point_buffer(
+    buffer: &mut SidecarActionsBuffer,
+    metric_name: CharSlice,
+    metric_value: f64,
+    integration_name: CharSlice,
+) {
+    let mut tags: Vec<Tag> = Vec::default();
+    if integration_name.len() > 0 {
+        tags.push(Tag::new("integration_name", integration_name.to_utf8_lossy().into_owned()).unwrap())
+    }
+
+    buffer.buffer.push(SidecarAction::AddTelemetryMetricPoint((
+        metric_name.to_utf8_lossy().into_owned(),
+        metric_value,
+        tags,
+    )));
+}

config.m4

@@ -306,7 +306,7 @@ EOT
     if test "$PHP_DDTRACE_SIDECAR_MOCKGEN" != "-"; then
       ddtrace_mockgen_invocation="HOST= TARGET= $PHP_DDTRACE_SIDECAR_MOCKGEN"
     else
-      ddtrace_mockgen_invocation="cd \"$ext_srcdir/components-rs/php_sidecar_mockgen\"; HOST= TARGET= CARGO_TARGET_DIR=\$(builddir)/target/ \$(DDTRACE_CARGO) run"
+      ddtrace_mockgen_invocation="cd \"$ext_srcdir/components-rs/php_sidecar_mockgen\"; HOST=$(rustc -vV | sed -n 's|host: ||p') TARGET=$(rustc -vV | sed -n 's|host: ||p') CARGO_TARGET_DIR=\$(builddir)/target/ \$(DDTRACE_CARGO) run"
     fi
     cat <<EOT >> Makefile.fragments
 

ext/ddtrace.c

@@ -419,6 +419,7 @@ static void ddtrace_activate(void) {
     zai_hook_rinit();
     zai_interceptor_activate();
     zai_uhook_rinit();
+    ddtrace_telemetry_rinit();
     zend_hash_init(&DDTRACE_G(traced_spans), 8, unused, NULL, 0);
     zend_hash_init(&DDTRACE_G(tracestate_unknown_dd_keys), 8, unused, NULL, 0);
 
@@ -1402,6 +1403,8 @@ static PHP_RSHUTDOWN_FUNCTION(ddtrace) {
     }
 
     dd_finalize_telemetry();
+    ddtrace_telemetry_rshutdown();
+
     if (DDTRACE_G(last_flushed_root_service_name)) {
         zend_string_release(DDTRACE_G(last_flushed_root_service_name));
         DDTRACE_G(last_flushed_root_service_name) = NULL;
@@ -1955,6 +1958,21 @@ PHP_FUNCTION(DDTrace_Testing_trigger_error) {
     }
 }
 
+PHP_FUNCTION(DDTrace_Internal_add_span_flag) {
+    zend_object *span;
+    zend_long flag;
+
+    ZEND_PARSE_PARAMETERS_START(2, 2)
+        Z_PARAM_OBJ_OF_CLASS_EX(span, ddtrace_ce_span_data, 0, 1)
+        Z_PARAM_LONG(flag)
+    ZEND_PARSE_PARAMETERS_END();
+
+    ddtrace_span_data *span_data = OBJ_SPANDATA(span);
+    span_data->flags |= (uint8_t)flag;
+
+    RETURN_NULL();
+}
+
 PHP_FUNCTION(ddtrace_init) {
     if (DDTRACE_G(request_init_hook_loaded) == 1) {
         RETURN_FALSE;

ext/ddtrace.h

@@ -109,6 +109,8 @@ ZEND_BEGIN_MODULE_GLOBALS(ddtrace)
     zend_string *last_flushed_root_service_name;
     zend_string *last_flushed_root_env_name;
 
+    HashTable telemetry_spans_created_per_integration;
+
     HashTable uhook_active_hooks;
     HashTable uhook_closure_hooks;
 ZEND_END_MODULE_GLOBALS(ddtrace)

ext/ddtrace.stub.php

@@ -669,6 +669,30 @@ function set_blocking_function(\DDTrace\RootSpanData $span, callable $blockingFu
     function trigger_error(string $message, int $errorType): void {}
 }
 
+namespace DDTrace\Internal {
+    /**
+     * @var int
+     * @cvalue DDTRACE_SPAN_FLAG_OPENTELEMETRY
+     */
+    const SPAN_FLAG_OPENTELEMETRY = UNKNOWN;
+
+    /**
+     * @var int
+     * @cvalue DDTRACE_SPAN_FLAG_OPENTRACING
+     */
+    const SPAN_FLAG_OPENTRACING = UNKNOWN;
+
+    /**
+     * Adds a flag to a span.
+     *
+     * @internal
+     *
+     * @param \DDTrace\SpanData $span the span to flag
+     * @param int $flag the flag to add to the span
+     */
+    function add_span_flag(\DDTrace\SpanData $span, int $flag): void {}
+}
+
 namespace {
 
     /**

ext/ddtrace_arginfo.h

@@ -1,5 +1,5 @@
 /* This is a generated file, edit the .stub.php file instead.
- * Stub hash: 7fec613751ec101cba339b6dfa3a72fbaf38074f */
+ * Stub hash: 1271064f2b0fcdc3d20e2f3d6c780607b300c5f9 */
 
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_DDTrace_trace_method, 0, 3, _IS_BOOL, 0)
 	ZEND_ARG_TYPE_INFO(0, className, IS_STRING, 0)
@@ -165,6 +165,11 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_DDTrace_Testing_trigger_error, 0
 	ZEND_ARG_TYPE_INFO(0, errorType, IS_LONG, 0)
 ZEND_END_ARG_INFO()
 
+ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_DDTrace_Internal_add_span_flag, 0, 2, IS_VOID, 0)
+	ZEND_ARG_OBJ_INFO(0, span, DDTrace\\SpanData, 0)
+	ZEND_ARG_TYPE_INFO(0, flag, IS_LONG, 0)
+ZEND_END_ARG_INFO()
+
 ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_dd_trace_env_config, 0, 1, IS_MIXED, 0)
 	ZEND_ARG_TYPE_INFO(0, envName, IS_STRING, 0)
 ZEND_END_ARG_INFO()
@@ -326,6 +331,7 @@ ZEND_FUNCTION(DDTrace_UserRequest_notify_start);
 ZEND_FUNCTION(DDTrace_UserRequest_notify_commit);
 ZEND_FUNCTION(DDTrace_UserRequest_set_blocking_function);
 ZEND_FUNCTION(DDTrace_Testing_trigger_error);
+ZEND_FUNCTION(DDTrace_Internal_add_span_flag);
 ZEND_FUNCTION(dd_trace_env_config);
 ZEND_FUNCTION(dd_trace_disable_in_request);
 ZEND_FUNCTION(dd_trace_reset);
@@ -409,6 +415,7 @@ static const zend_function_entry ext_functions[] = {
 	ZEND_NS_FALIAS("DDTrace\\UserRequest", notify_commit, DDTrace_UserRequest_notify_commit, arginfo_DDTrace_UserRequest_notify_commit)
 	ZEND_NS_FALIAS("DDTrace\\UserRequest", set_blocking_function, DDTrace_UserRequest_set_blocking_function, arginfo_DDTrace_UserRequest_set_blocking_function)
 	ZEND_NS_FALIAS("DDTrace\\Testing", trigger_error, DDTrace_Testing_trigger_error, arginfo_DDTrace_Testing_trigger_error)
+	ZEND_NS_FALIAS("DDTrace\\Internal", add_span_flag, DDTrace_Internal_add_span_flag, arginfo_DDTrace_Internal_add_span_flag)
 	ZEND_FE(dd_trace_env_config, arginfo_dd_trace_env_config)
 	ZEND_FE(dd_trace_disable_in_request, arginfo_dd_trace_disable_in_request)
 	ZEND_FE(dd_trace_reset, arginfo_dd_trace_reset)
@@ -479,6 +486,8 @@ static void register_ddtrace_symbols(int module_number)
 	REGISTER_LONG_CONSTANT("DDTrace\\DBM_PROPAGATION_DISABLED", DD_TRACE_DBM_PROPAGATION_DISABLED, CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("DDTrace\\DBM_PROPAGATION_SERVICE", DD_TRACE_DBM_PROPAGATION_SERVICE, CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("DDTrace\\DBM_PROPAGATION_FULL", DD_TRACE_DBM_PROPAGATION_FULL, CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("DDTrace\\Internal\\SPAN_FLAG_OPENTELEMETRY", DDTRACE_SPAN_FLAG_OPENTELEMETRY, CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("DDTrace\\Internal\\SPAN_FLAG_OPENTRACING", DDTRACE_SPAN_FLAG_OPENTRACING, CONST_PERSISTENT);
 	REGISTER_STRING_CONSTANT("DD_TRACE_VERSION", PHP_DDTRACE_VERSION, CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("DD_TRACE_PRIORITY_SAMPLING_AUTO_KEEP", PRIORITY_SAMPLING_AUTO_KEEP, CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("DD_TRACE_PRIORITY_SAMPLING_AUTO_REJECT", PRIORITY_SAMPLING_AUTO_REJECT, CONST_PERSISTENT);

ext/span.c

@@ -12,6 +12,7 @@
 #include <components/log/log.h>
 #include "random.h"
 #include "serializer.h"
+#include "telemetry.h"
 #include "ext/standard/php_string.h"
 #include <hook/hook.h>
 #include "user_request.h"
@@ -553,6 +554,10 @@ void ddtrace_close_span(ddtrace_span_data *span) {
         ddtrace_switch_span_stack(span->stack);
     }
 
+    // Telemetry: increment the spans_created counter
+    // Must be done at closing because we need to read the "component" span's meta which is not available at creation
+    ddtrace_telemetry_inc_spans_created(span);
+
     ddtrace_close_stack_userland_spans_until(span);
 
     ddtrace_close_top_span_without_stack_swap(span);

ext/span.h

@@ -13,6 +13,9 @@
 #define DDTRACE_DROPPED_SPAN (-1ull)
 #define DDTRACE_SILENTLY_DROPPED_SPAN (-2ull)
 
+#define DDTRACE_SPAN_FLAG_OPENTELEMETRY (1 << 0)
+#define DDTRACE_SPAN_FLAG_OPENTRACING (1 << 1)
+
 struct ddtrace_span_stack;
 
 enum ddtrace_span_dataype {
@@ -72,6 +75,7 @@ struct ddtrace_span_data {
     uint64_t start;
     uint64_t duration_start;
     uint64_t duration;
+    uint8_t flags;
     enum ddtrace_span_dataype type : 8;
     bool notify_user_req_end;
     struct ddtrace_span_data *next;

ext/telemetry.c

@@ -25,6 +25,14 @@ void ddtrace_telemetry_first_init(void) {
     dd_composer_hook_id = zai_hook_install((zai_str)ZAI_STR_EMPTY, (zai_str)ZAI_STR_EMPTY, dd_check_for_composer_autoloader, NULL, ZAI_HOOK_AUX_UNUSED, 0);
 }
 
+void ddtrace_telemetry_rinit(void) {
+    zend_hash_init(&DDTRACE_G(telemetry_spans_created_per_integration), 8, unused, NULL, 0);
+}
+
+void ddtrace_telemetry_rshutdown(void) {
+    zend_hash_destroy(&DDTRACE_G(telemetry_spans_created_per_integration));
+}
+
 void ddtrace_telemetry_finalize(void) {
     if (!ddtrace_sidecar || !get_global_DD_INSTRUMENTATION_TELEMETRY_ENABLED()) {
         return;
@@ -69,6 +77,16 @@ void ddtrace_telemetry_finalize(void) {
             ddog_sidecar_telemetry_addIntegration_buffer(buffer, integration_name, DDOG_CHARSLICE_C(""), false);
         }
     }
+
+    // Telemetry metrics
+    ddog_CharSlice metric_name = DDOG_CHARSLICE_C("spans_created");
+    ddog_sidecar_telemetry_register_metric_buffer(buffer, metric_name);
+    zend_string *integration_name;
+    zval *metric_value;
+    ZEND_HASH_FOREACH_STR_KEY_VAL(&DDTRACE_G(telemetry_spans_created_per_integration), integration_name, metric_value) {
+        ddog_sidecar_telemetry_add_span_metric_point_buffer(buffer, metric_name, Z_DVAL_P(metric_value), dd_zend_string_to_CharSlice(integration_name));
+    } ZEND_HASH_FOREACH_END();
+
     ddog_sidecar_telemetry_buffer_flush(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(telemetry_queue_id), buffer);
 
     ddog_CharSlice service_name = DDOG_CHARSLICE_C_BARE("unnamed-php-service");
@@ -88,7 +106,7 @@ void ddtrace_telemetry_finalize(void) {
 
     ddog_sidecar_runtimeMeta_drop(meta);
 
-    ddog_sidecar_telemetry_end(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(telemetry_queue_id));
+    //ddog_sidecar_telemetry_end(&ddtrace_sidecar, ddtrace_sidecar_instance_id, &DDTRACE_G(telemetry_queue_id));
 }
 
 void ddtrace_telemetry_notify_integration(const char *name, size_t name_len) {
@@ -98,3 +116,33 @@ void ddtrace_telemetry_notify_integration(const char *name, size_t name_len) {
                                               DDOG_CHARSLICE_C(""), true);
     }
 }
+
+void ddtrace_telemetry_inc_spans_created(ddtrace_span_data *span) {
+    zval *component = NULL;
+    if (Z_TYPE(span->property_meta) == IS_ARRAY) {
+        component = zend_hash_str_find(Z_ARRVAL(span->property_meta), ZEND_STRL("component"));
+    }
+
+    zend_string *integration = NULL;
+    if (component && Z_TYPE_P(component) == IS_STRING) {
+        integration = zend_string_copy(Z_STR_P(component));
+    } else if (span->flags & DDTRACE_SPAN_FLAG_OPENTELEMETRY) {
+        integration = zend_string_init(ZEND_STRL("otel"), 0);
+    } else if (span->flags & DDTRACE_SPAN_FLAG_OPENTRACING) {
+        integration = zend_string_init(ZEND_STRL("opentracing"), 0);
+    } else {
+        // Fallback value when the span has not been created by an integration, nor OpenTelemetry/OpenTracing (i.e. \DDTrace\span_start())
+        integration = zend_string_init(ZEND_STRL("datadog"), 0);
+    }
+
+    zval *current = zend_hash_find(&DDTRACE_G(telemetry_spans_created_per_integration), integration);
+    if (current) {
+        ++Z_DVAL_P(current);
+    } else {
+        zval counter;
+        ZVAL_DOUBLE(&counter, 1.0);
+        zend_hash_add(&DDTRACE_G(telemetry_spans_created_per_integration), integration, &counter);
+    }
+
+    zend_string_release(integration);
+}

ext/telemetry.h

@@ -4,8 +4,10 @@
 #include <components-rs/telemetry.h>
 
 void ddtrace_telemetry_first_init(void);
+void ddtrace_telemetry_rinit(void);
+void ddtrace_telemetry_rshutdown(void);
 ddog_TelemetryWorkerHandle *ddtrace_build_telemetry_handle(void);
 void ddtrace_telemetry_notify_integration(const char *name, size_t name_len);
 void ddtrace_telemetry_finalize(void);
-
+void ddtrace_telemetry_inc_spans_created(ddtrace_span_data *span);
 #endif // DDTRACE_TELEMETRY_H