Skip to content

Commit

Permalink
OtelTraceProvider.Builder introduce the trace rate limit property
Browse files Browse the repository at this point in the history
  • Loading branch information
@mariusc83
mariusc83 committed Jun 14, 2024
1 parent 98c43e2 commit d523380
Show file tree
Hide file tree
Showing 5 changed files with 235 additions and 6 deletions.
1 change: 1 addition & 0 deletions features/dd-sdk-android-trace-otel/api/apiSurface
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,5 +10,6 @@ class com.datadog.android.trace.opentelemetry.OtelTracerProvider : io.openteleme
fun setPartialFlushThreshold(Int): Builder
fun addTag(String, String): Builder
fun setSampleRate(Double): Builder
fun setTraceRateLimit(Int): Builder
fun setBundleWithRumEnabled(Boolean): Builder
override fun toString(): String
1 change: 1 addition & 0 deletions features/dd-sdk-android-trace-otel/api/dd-sdk-android-trace-otel.api
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public final class com/datadog/android/trace/opentelemetry/OtelTracerProvider$Bu
public final fun setPartialFlushThreshold (I)Lcom/datadog/android/trace/opentelemetry/OtelTracerProvider$Builder;
public final fun setSampleRate (D)Lcom/datadog/android/trace/opentelemetry/OtelTracerProvider$Builder;
public final fun setService (Ljava/lang/String;)Lcom/datadog/android/trace/opentelemetry/OtelTracerProvider$Builder;
public final fun setTraceRateLimit (I)Lcom/datadog/android/trace/opentelemetry/OtelTracerProvider$Builder;
public final fun setTracingHeaderTypes (Ljava/util/Set;)Lcom/datadog/android/trace/opentelemetry/OtelTracerProvider$Builder;
}

Expand Down
17 changes: 17 additions & 0 deletions ...-trace-otel/src/main/kotlin/com/datadog/android/trace/opentelemetry/OtelTracerProvider.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package com.datadog.android.trace.opentelemetry

import androidx.annotation.FloatRange
import androidx.annotation.IntRange
import com.datadog.android.Datadog
import com.datadog.android.api.InternalLogger
import com.datadog.android.api.SdkCore
Expand Down Expand Up @@ -105,6 +106,8 @@ class OtelTracerProvider internal constructor(
private var tracingHeaderTypes: Set<TracingHeaderType> =
setOf(TracingHeaderType.DATADOG, TracingHeaderType.TRACECONTEXT)
private var sampleRate: Double? = null
private var traceRateLimit = Int.MAX_VALUE

private var serviceName: String = ""
get() {
return field.ifEmpty {
Expand Down Expand Up @@ -236,6 +239,19 @@ class OtelTracerProvider internal constructor(
return this
}

/**
* Sets the trace rate limit. This is the maximum number of traces per second that will be
* accepted. Please not that this property is used in conjunction with the sample rate. If no sample rate
* is provided this property and its related logic will be ignored.
* @param traceRateLimit the trace rate limit as a value between 0 and Int.MAX_VALUE (default is Int.MAX_VALUE)
*/
fun setTraceRateLimit(
@IntRange(from = 0, to = Int.MAX_VALUE.toLong()) traceRateLimit: Int
): Builder {
this.traceRateLimit = traceRateLimit
return this
}

/**
* Enables the trace bundling with the current active View. If this feature is enabled all
* the spans from this moment on will be bundled with the current view information and you
Expand All @@ -253,6 +269,7 @@ class OtelTracerProvider internal constructor(
TracerConfig.SPAN_TAGS,
globalTags.map { "${it.key}:${it.value}" }.joinToString(",")
)
properties.setProperty(TracerConfig.TRACE_RATE_LIMIT, traceRateLimit.toString())

// In case the sample rate is not set we should not specify it. The agent code under the hood
// will provide different sampler based on this property and also different sampling priorities used
Expand Down
33 changes: 33 additions & 0 deletions .../src/test/kotlin/com/datadog/android/trace/opentelemetry/OtelTracerBuilderProviderTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -631,6 +631,39 @@ internal class OtelTracerBuilderProviderTest {

// endregion

// region trace rate limit

@Test
fun `M use the trace rate limit W setTraceRateLimit`(
@IntForgery(min = 0, max = Int.MAX_VALUE) traceRateLimit: Int
) {
// Given
val tracer = testedOtelTracerProviderBuilder.setTraceRateLimit(traceRateLimit).build()
.tracerBuilder(fakeInstrumentationName).build()

// When
val coreTracer: CoreTracer = tracer.getFieldValue("tracer")

// Then
val config: Config = coreTracer.getFieldValue("initialConfig")
assertThat(config.traceRateLimit).isEqualTo(traceRateLimit)
}

@Test
fun `M use the default rate limit W build { if not provided }`() {
// Given
val tracer = testedOtelTracerProviderBuilder.build().tracerBuilder(fakeInstrumentationName).build()

// When
val coreTracer: CoreTracer = tracer.getFieldValue("tracer")

// Then
val config: Config = coreTracer.getFieldValue("initialConfig")
assertThat(config.traceRateLimit).isEqualTo(Int.MAX_VALUE)
}

// endregion

// region bundle with RUM

@Test
Expand Down
189 changes: 183 additions & 6 deletions ...race/src/test/kotlin/com/datadog/android/trace/integration/otel/OtelTracerProviderTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import com.google.gson.JsonObject
import com.google.gson.JsonParser
import fr.xgouchet.elmyr.Forge
import fr.xgouchet.elmyr.annotation.DoubleForgery
import fr.xgouchet.elmyr.annotation.IntForgery
import fr.xgouchet.elmyr.annotation.StringForgery
import fr.xgouchet.elmyr.junit5.ForgeConfiguration
import fr.xgouchet.elmyr.junit5.ForgeExtension
Expand Down Expand Up @@ -769,7 +770,7 @@ internal class OtelTracerProviderTest {
}
}

@Test
@RepeatedTest(10)
fun `M use user-keep or user-drop priority W buildSpan { tracer was provided a sample rate }`(
@StringForgery fakeInstrumentationName: String,
@DoubleForgery(min = 0.0, max = 100.0) sampleRate: Double,
Expand All @@ -786,11 +787,9 @@ internal class OtelTracerProviderTest {

// When
repeat(numberOfSpans) {
val span = tracer.spanBuilder(forge.anAlphabeticalString()).startSpan()
// there is a throttle on the sampler which drops all the spans over the 100 limit in 1 second
// so we need to sleep a bit to make sure the spans are not dropped because of throttling
Thread.sleep(10)
span.end()
tracer.spanBuilder(forge.anAlphabeticalString())
.startSpan()
.end()
}

// Then
Expand Down Expand Up @@ -818,6 +817,183 @@ internal class OtelTracerProviderTest {

// endregion

// region trace rate limit

@Test
fun `M drop the spans W buildSpan { trace rate limit reached in 1 second, sample rate specified }`(
@StringForgery fakeInstrumentationName: String,
@IntForgery(min = 1, max = 3) traceLimit: Int,
forge: Forge
) {
// Given
val testedProvider = OtelTracerProvider.Builder(stubSdkCore)
.setTraceRateLimit(traceLimit)
.setSampleRate(100.0)
.build()
val tracer = testedProvider.tracerBuilder(fakeInstrumentationName).build()
val blockingWriterWrapper = tracer.useBlockingWriter()

// When
val startNanos = System.nanoTime()
var spansCounter = 0
while ((System.nanoTime() - startNanos) < ONE_SECOND_AS_NANOS && (spansCounter < 200)) {
tracer.spanBuilder(forge.anAlphabeticalString()).startSpan().end()
spansCounter++
}

// Then
blockingWriterWrapper.waitForTracesMax(spansCounter)
val spansWritten = stubSdkCore.eventsWritten(Feature.TRACING_FEATURE_NAME)
.map {
(JsonParser.parseString(it.eventData) as JsonObject)
.getAsJsonArray("spans")
.get(0)
.asJsonObject
}
val userKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_KEEP.toInt()
}
val samplerKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_KEEP.toInt()
}
assertThat(samplerKeptSpans.size).isEqualTo(0)
assertThat(userKeptSpans.size).isLessThanOrEqualTo(traceLimit)
}

@Test
fun `M ignore trace rate limit W buildSpan { trace rate limit reached in 1 second, sample rate not specified }`(
@StringForgery fakeInstrumentationName: String,
@IntForgery(min = 1, max = 3) traceLimit: Int,
forge: Forge
) {
// Given
val testedProvider = OtelTracerProvider.Builder(stubSdkCore)
.setTraceRateLimit(traceLimit)
.build()
val tracer = testedProvider.tracerBuilder(fakeInstrumentationName).build()
val blockingWriterWrapper = tracer.useBlockingWriter()

// When
val startNanos = System.nanoTime()
var spansCounter = 0
while ((System.nanoTime() - startNanos) < ONE_SECOND_AS_NANOS && (spansCounter < 200)) {
tracer.spanBuilder(forge.anAlphabeticalString()).startSpan().end()
spansCounter++
}

// Then
blockingWriterWrapper.waitForTracesMax(spansCounter)
val spansWritten = stubSdkCore.eventsWritten(Feature.TRACING_FEATURE_NAME)
.map {
(JsonParser.parseString(it.eventData) as JsonObject)
.getAsJsonArray("spans")
.get(0)
.asJsonObject
}
val userKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_KEEP.toInt()
}
val samplerKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_KEEP.toInt()
}
assertThat(samplerKeptSpans.size).isEqualTo(spansCounter)
assertThat(userKeptSpans.size).isEqualTo(0)
}

@Test
fun `M ignore trace limit W buildSpan { trace rate limit is 0 but sample rate is not specified }`(
@StringForgery fakeInstrumentationName: String,
forge: Forge
) {
// Given
val testedProvider = OtelTracerProvider.Builder(stubSdkCore).setTraceRateLimit(0).build()
val tracer = testedProvider.tracerBuilder(fakeInstrumentationName).build()
val blockingWriterWrapper = tracer.useBlockingWriter()

// When
val startNanos = System.nanoTime()
var spansCounter = 0
while (((System.nanoTime() - startNanos) < (ONE_SECOND_AS_NANOS * 2)) && (spansCounter < 200)) {
tracer.spanBuilder(forge.anAlphabeticalString()).startSpan().end()
spansCounter++
}

// Then
blockingWriterWrapper.waitForTracesMax(spansCounter)
val spansWritten = stubSdkCore.eventsWritten(Feature.TRACING_FEATURE_NAME)
.map {
(JsonParser.parseString(it.eventData) as JsonObject)
.getAsJsonArray("spans")
.get(0)
.asJsonObject
}
val userDroppedSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_DROP.toInt()
}
val samplerDroppedSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_DROP.toInt()
}
val userKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_KEEP.toInt()
}
val keptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_KEEP.toInt()
}
assertThat(userDroppedSpans.size).isEqualTo(0)
assertThat(samplerDroppedSpans.size).isEqualTo(0)
assertThat(userKeptSpans.size).isEqualTo(0)
assertThat(keptSpans.size).isEqualTo(spansCounter)
}

@Test
fun `M only keep 1 span (default trace limit) W buildSpan { trace rate limit is 0 and sample rate is specified }`(
@StringForgery fakeInstrumentationName: String,
@IntForgery(min = 2, max = 10) numberOfSpans: Int,
forge: Forge
) {
// Given
val testedProvider = OtelTracerProvider.Builder(stubSdkCore)
.setTraceRateLimit(0)
.setSampleRate(100.0)
.build()
val tracer = testedProvider.tracerBuilder(fakeInstrumentationName).build()
val blockingWriterWrapper = tracer.useBlockingWriter()

// When
repeat(numberOfSpans) {
tracer.spanBuilder(forge.anAlphabeticalString()).startSpan().end()
}

// Then
blockingWriterWrapper.waitForTracesMax(numberOfSpans)
val spansWritten = stubSdkCore.eventsWritten(Feature.TRACING_FEATURE_NAME)
.map {
(JsonParser.parseString(it.eventData) as JsonObject)
.getAsJsonArray("spans")
.get(0)
.asJsonObject
}
val userDroppedSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_DROP.toInt()
}
val samplerDroppedSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_DROP.toInt()
}
val userKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.USER_KEEP.toInt()
}
val samplerKeptSpans = spansWritten.filter {
it.getInt(SAMPLING_PRIORITY_KEY) == PrioritySampling.SAMPLER_KEEP.toInt()
}
assertThat(userDroppedSpans.size + userKeptSpans.size).isEqualTo(numberOfSpans)
assertThat(samplerDroppedSpans.size).isEqualTo(0)
assertThat(userKeptSpans.size).isEqualTo(1)
assertThat(samplerKeptSpans.size).isEqualTo(0)
assertThat(userDroppedSpans.size).isEqualTo(numberOfSpans - 1)
}

// endregion

// region Bundle with RUM

@RepeatedTest(10)
Expand Down Expand Up @@ -972,6 +1148,7 @@ internal class OtelTracerProviderTest {
// endregion

companion object {
private val ONE_SECOND_AS_NANOS = TimeUnit.SECONDS.toNanos(1)
private const val DEFAULT_SPAN_NAME = "internal"
private const val JOIN_TIMEOUT_MS = 5000L
private const val SAMPLING_PRIORITY_KEY = "metrics._sampling_priority_v1"
Expand Down

0 comments on commit d523380

Please sign in to comment.