reduced allocation v04 span representation

LICENSE-3rdparty.yml

@@ -1,4 +1,4 @@
-root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent, bin_tests
+root_name: datadog-alloc, builder, build_common, datadog-profiling-ffi, data-pipeline-ffi, data-pipeline, datadog-ddsketch, datadog-trace-normalization, datadog-trace-protobuf, datadog-trace-utils, ddcommon, tinybytes, ddcommon-ffi, datadog-crashtracker-ffi, datadog-crashtracker, ddtelemetry, datadog-profiling, ddtelemetry-ffi, symbolizer-ffi, tools, datadog-profiling-replayer, dogstatsd, datadog-ipc, datadog-ipc-macros, tarpc, tarpc-plugins, spawn_worker, cc_utils, datadog-sidecar, datadog-remote-config, datadog-dynamic-configuration, datadog-sidecar-macros, datadog-sidecar-ffi, sidecar_mockgen, datadog-trace-obfuscation, test_spawn_from_lib, datadog-serverless-trace-mini-agent, datadog-trace-mini-agent
 third_party_libraries:
 - package_name: addr2line
   package_version: 0.21.0
@@ -24432,7 +24432,7 @@ third_party_libraries:
   - license: BSD-3-Clause
     text: NOT FOUND
 - package_name: serde
-  package_version: 1.0.197
+  package_version: 1.0.209
   repository: https://github.com/serde-rs/serde
   license: MIT OR Apache-2.0
   licenses:
@@ -24848,7 +24848,7 @@ third_party_libraries:
 
       END OF TERMS AND CONDITIONS
 - package_name: serde_derive
-  package_version: 1.0.197
+  package_version: 1.0.209
   repository: https://github.com/serde-rs/serde
   license: MIT OR Apache-2.0
   licenses:
@@ -25056,7 +25056,7 @@ third_party_libraries:
 
       END OF TERMS AND CONDITIONS
 - package_name: serde_json
-  package_version: 1.0.120
+  package_version: 1.0.127
   repository: https://github.com/serde-rs/json
   license: MIT OR Apache-2.0
   licenses:

data-pipeline/src/trace_exporter.rs

@@ -5,7 +5,7 @@ use bytes::Bytes;
 use datadog_trace_protobuf::pb;
 use datadog_trace_utils::trace_utils::{self, SendData, TracerHeaderTags};
 use datadog_trace_utils::tracer_payload;
-use datadog_trace_utils::tracer_payload::TraceEncoding;
+use datadog_trace_utils::tracer_payload::TraceCollection;
 use ddcommon::{connector, Endpoint};
 use hyper::http::uri::PathAndQuery;
 use hyper::{Body, Client, Method, Uri};
@@ -231,11 +231,10 @@ impl TraceExporter {
             ),
             TraceExporterOutputFormat::V07 => {
                 let tracer_payload = trace_utils::collect_trace_chunks(
-                    traces,
+                    TraceCollection::V07(traces),
                     &header_tags,
                     &mut tracer_payload::DefaultTraceChunkProcessor,
                     self.endpoint.api_key.is_some(),
-                    TraceEncoding::V07,
                 );
 
                 let endpoint = Endpoint {

ipc/Cargo.toml

@@ -17,6 +17,7 @@ serde = { version = "1.0", default-features = false, features = ["derive"] }
 tokio-serde = { version = "0.8", features = ["bincode"] }
 tokio-util = { version = "0.6.9", features = ["codec"] }
 libc = { version = "0.2" }
+tinybytes = { path = "../tinybytes", optional = true }
 
 # tarpc needed extensions to allow 1 way communication and to export some internal structs
 tarpc = { path = "tarpc/tarpc", default-features = false, features = ["serde-transport"], package = "tarpc" }
@@ -63,3 +64,6 @@ path = "benches/ipc.rs"
 
 [lints.rust]
 unexpected_cfgs = { level = "warn", check-cfg = ['cfg(polyfill_glibc_memfd)'] }
+
+[features]
+tiny-bytes = ["tinybytes"]

ipc/src/platform/mem_handle.rs

@@ -7,6 +7,8 @@ use serde::{Deserialize, Serialize};
 #[cfg(all(unix, not(target_os = "macos")))]
 use std::os::unix::prelude::AsRawFd;
 use std::{ffi::CString, io};
+#[cfg(feature = "tiny-bytes")]
+use tinybytes::UnderlyingBytes;
 
 #[derive(Clone, Serialize, Deserialize, Debug)]
 pub struct ShmHandle {
@@ -203,6 +205,15 @@ impl From<ShmHandle> for PlatformHandle<OwnedFileHandle> {
 unsafe impl<T> Sync for MappedMem<T> where T: FileBackedHandle {}
 unsafe impl<T> Send for MappedMem<T> where T: FileBackedHandle {}
 
+impl AsRef<[u8]> for MappedMem<ShmHandle> {
+    fn as_ref(&self) -> &[u8] {
+        self.as_slice()
+    }
+}
+
+#[cfg(feature = "tiny-bytes")]
+impl UnderlyingBytes for MappedMem<ShmHandle> {}
+
 #[cfg(test)]
 mod tests {
     use crate::platform::{FileBackedHandle, NamedShmHandle, ShmHandle};

sidecar/Cargo.toml

@@ -26,6 +26,7 @@ datadog-trace-utils = { path = "../trace-utils" }
 datadog-trace-normalization = { path = "../trace-normalization" }
 datadog-remote-config = { path = "../remote-config" }
 datadog-crashtracker = { path = "../crashtracker" }
+tinybytes = { path = "../tinybytes" }
 
 futures = { version = "0.3", default-features = false }
 manual_future = "0.1.1"
@@ -34,7 +35,7 @@ hyper = { version = "0.14", features = ["client"], default-features = false }
 lazy_static = "1.4"
 pin-project = "1"
 
-datadog-ipc = { path = "../ipc" }
+datadog-ipc = { path = "../ipc", features = ["tiny-bytes"] }
 datadog-ipc-macros = { path = "../ipc/macros" }
 
 rand = "0.8.3"

sidecar/src/service/sidecar_server.rs

@@ -47,6 +47,7 @@ use datadog_ipc::platform::FileBackedHandle;
 use datadog_ipc::tarpc::server::{Channel, InFlightRequest};
 use datadog_remote_config::fetch::ConfigInvariants;
 use datadog_trace_utils::tracer_header_tags::TracerHeaderTags;
+use tinybytes;
 
 type NoResponse = Ready<()>;
 
@@ -248,7 +249,12 @@ impl SidecarServer {
             .expect("Unable to acquire lock on sessions")
     }
 
-    fn send_trace_v04(&self, headers: &SerializedTracerHeaderTags, data: &[u8], target: &Endpoint) {
+    fn send_trace_v04(
+        &self,
+        headers: &SerializedTracerHeaderTags,
+        data: tinybytes::Bytes,
+        target: &Endpoint,
+    ) {
         let headers: TracerHeaderTags = match headers.try_into() {
             Ok(headers) => headers,
             Err(e) => {
@@ -741,7 +747,7 @@ impl SidecarInterface for SidecarServer {
         _: Context,
         instance_id: InstanceId,
         handle: ShmHandle,
-        len: usize,
+        _len: usize,
         headers: SerializedTracerHeaderTags,
     ) -> Self::SendTraceV04ShmFut {
         if let Some(endpoint) = self
@@ -753,7 +759,8 @@ impl SidecarInterface for SidecarServer {
             tokio::spawn(async move {
                 match handle.map() {
                     Ok(mapped) => {
-                        self.send_trace_v04(&headers, &mapped.as_slice()[..len], &endpoint);
+                        let bytes = tinybytes::Bytes::from(mapped);
+                        self.send_trace_v04(&headers, bytes, &endpoint);
                     }
                     Err(e) => error!("Failed mapping shared trace data memory: {}", e),
                 }
@@ -779,7 +786,8 @@ impl SidecarInterface for SidecarServer {
             .clone()
         {
             tokio::spawn(async move {
-                self.send_trace_v04(&headers, data.as_slice(), &endpoint);
+                let bytes = tinybytes::Bytes::from(data);
+                self.send_trace_v04(&headers, bytes, &endpoint);
             });
         }
 

tinybytes/Cargo.toml

@@ -14,3 +14,14 @@ once_cell = "1.8"
 pretty_assertions = "1.3"
 proptest = {version = "1.5", features = ["std"], default-features = false}
 test-case = "2.2"
+serde_json = "1.0.127"
+tinybytes = { path = ".", features = ["bytes_string", "serialization"] }
+
+[dependencies]
+serde = { version = "1.0.209", optional = true }
+#datadog-ipc = { path = "../ipc", optional = true }
+
+[features]
+bytes_string = []
+serialization = ["serde"]
+#ipc = ["datadog-ipc"]