Skip to content

Commit

Permalink
Add an option to limit stack depth (#739)
Browse files Browse the repository at this point in the history 
* Add an option to limit stack depth

* Fix compile error

* rename stack depth configuration property

* fix build

* spotless

* address review comments
  • Loading branch information
@laurit
laurit authored Apr 20, 2022
1 parent d1198c5 commit f685519
Show file tree
Hide file tree
Showing 14 changed files with 160 additions and 31 deletions.
2 changes: 2 additions & 0 deletions profiler/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ dependencies {
compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
compileOnly("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api")
compileOnly("io.opentelemetry:opentelemetry-semconv")
// required to access InstrumentationHolder
compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap")
implementation("io.opentelemetry:opentelemetry-sdk-logs")
implementation("io.opentelemetry:opentelemetry-exporter-otlp-logs")
implementation("com.google.protobuf:protobuf-java:$protobufVersion")
Expand Down
10 changes: 8 additions & 2 deletions profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ public class Configuration implements ConfigPropertySource {
public static final Duration DEFAULT_CALL_STACK_INTERVAL = Duration.ofSeconds(10);
public static final boolean DEFAULT_INCLUDE_INTERNAL_STACKS = false;
public static final boolean DEFAULT_TRACING_STACKS_ONLY = false;
private static final int DEFAULT_STACK_DEPTH = 1024;

public static final String CONFIG_KEY_ENABLE_PROFILER = PROFILER_ENABLED_PROPERTY;
public static final String CONFIG_KEY_PROFILER_DIRECTORY = "splunk.profiler.directory";
Expand All @@ -46,8 +47,8 @@ public class Configuration implements ConfigPropertySource {
public static final String CONFIG_KEY_MEMORY_ENABLED = PROFILER_MEMORY_ENABLED_PROPERTY;
public static final String CONFIG_KEY_MEMORY_SAMPLER_INTERVAL =
"splunk.profiler.memory.sampler.interval";
public static final String CONFIG_KEY_CPU_DATA_FORMAT = "splunk.profiler.cpu.data.format";
public static final String CONFIG_KEY_MEMORY_DATA_FORMAT = "splunk.profiler.memory.data.format";
private static final String CONFIG_KEY_CPU_DATA_FORMAT = "splunk.profiler.cpu.data.format";
private static final String CONFIG_KEY_MEMORY_DATA_FORMAT = "splunk.profiler.memory.data.format";
public static final String CONFIG_KEY_TLAB_ENABLED = "splunk.profiler.tlab.enabled";
public static final String CONFIG_KEY_CALL_STACK_INTERVAL = "splunk.profiler.call.stack.interval";
public static final String CONFIG_KEY_DEPRECATED_THREADDUMP_PERIOD =
Expand All @@ -60,6 +61,7 @@ public class Configuration implements ConfigPropertySource {
public static final String CONFIG_KEY_INCLUDE_INTERNAL_STACKS =
"splunk.profiler.include.internal.stacks";
public static final String CONFIG_KEY_TRACING_STACKS_ONLY = "splunk.profiler.tracing.stacks.only";
private static final String CONFIG_KEY_STACK_DEPTH = "splunk.profiler.max.stack.depth";

@Override
public Map<String, String> getProperties() {
Expand Down Expand Up @@ -109,6 +111,10 @@ public static boolean getTracingStacksOnly(Config config) {
return config.getBoolean(CONFIG_KEY_TRACING_STACKS_ONLY, DEFAULT_TRACING_STACKS_ONLY);
}

public static int getStackDepth(Config config) {
return config.getInt(CONFIG_KEY_STACK_DEPTH, DEFAULT_STACK_DEPTH);
}

public static DataFormat getCpuDataFormat(Config config) {
String value = config.getString(CONFIG_KEY_CPU_DATA_FORMAT, "text");
return getDataFormat(value);
Expand Down
45 changes: 45 additions & 0 deletions profiler/src/main/java/com/splunk/opentelemetry/profiler/JFR.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,15 +16,24 @@

package com.splunk.opentelemetry.profiler;

import io.opentelemetry.javaagent.bootstrap.InstrumentationHolder;
import java.io.IOException;
import java.lang.instrument.Instrumentation;
import java.nio.file.Path;
import java.util.Collections;
import jdk.jfr.FlightRecorder;
import jdk.jfr.Recording;
import jdk.jfr.consumer.RecordedEvent;
import jdk.jfr.consumer.RecordingFile;
import jdk.jfr.internal.Options;
import net.bytebuddy.dynamic.loading.ClassInjector;
import net.bytebuddy.utility.JavaModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/** Abstraction around the JDK Flight Recorder subsystem. */
class JFR {
private static final Logger logger = LoggerFactory.getLogger(JFR.class);

public static final JFR instance = new JFR();
private static final boolean jfrAvailable = checkJfr();
Expand Down Expand Up @@ -61,4 +70,40 @@ public RecordedEvent readEvent(RecordingFile file, Path path) {
throw new JfrException("Error reading events from " + path, e);
}
}

public void setStackDepth(int stackDepth) {
try {
JfrAccess.init();
int currentDepth = Options.getStackDepth();
// we only increase stack depth
if (currentDepth < stackDepth) {
Options.setStackDepth(stackDepth);
}
} catch (Throwable throwable) {
logger.warn("Failed to set JFR stack depth to {}", stackDepth, throwable);
}
}

private static class JfrAccess {
static {
Instrumentation instrumentation = InstrumentationHolder.getInstrumentation();
if (instrumentation != null && JavaModule.isSupported()) {
// ensure that we have access to jdk.jfr.internal.Options.setStackDepth
JavaModule currentModule = JavaModule.ofType(JFR.class);
JavaModule flightRecorder = JavaModule.ofType(FlightRecorder.class);
if (flightRecorder != null && flightRecorder.isNamed() && currentModule != null) {
ClassInjector.UsingInstrumentation.redefineModule(
instrumentation,
flightRecorder,
Collections.emptySet(),
Collections.emptyMap(),
Collections.singletonMap("jdk.jfr.internal", Collections.singleton(currentModule)),
Collections.emptySet(),
Collections.emptyMap());
}
}
}

static void init() {}
}
}
6 changes: 6 additions & 0 deletions profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,9 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
Path outputDir = Paths.get(config.getString(CONFIG_KEY_PROFILER_DIRECTORY));
RecordingFileNamingConvention namingConvention = new RecordingFileNamingConvention(outputDir);

int stackDepth = Configuration.getStackDepth(config);
JFR.instance.setStackDepth(stackDepth);

RecordingEscapeHatch recordingEscapeHatch =
RecordingEscapeHatch.builder()
.namingConvention(namingConvention)
Expand Down Expand Up @@ -122,6 +125,7 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
.resource(resource)
.dataFormat(cpuDataFormat)
.eventPeriods(periods)
.stackDepth(stackDepth)
.build();
}

Expand All @@ -137,13 +141,15 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
.logProcessor(BatchLogProcessorHolder.get(logsExporter))
.commonAttributes(commonAttributes)
.resource(resource)
.stackDepth(stackDepth)
.build();
} else {
allocationEventExporter =
PprofAllocationEventExporter.builder()
.logProcessor(SimpleLogProcessor.create(logsExporter))
.resource(resource)
.dataFormat(allocationDataFormat)
.stackDepth(stackDepth)
.build();
}

Expand Down
3 changes: 3 additions & 0 deletions profiler/src/main/java/com/splunk/opentelemetry/profiler/ProfilingSemanticAttributes.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.splunk.opentelemetry.profiler;

import static io.opentelemetry.api.common.AttributeKey.booleanKey;
import static io.opentelemetry.api.common.AttributeKey.longKey;
import static io.opentelemetry.api.common.AttributeKey.stringKey;

Expand All @@ -42,6 +43,8 @@ public class ProfilingSemanticAttributes {
public static final AttributeKey<String> THREAD_NAME = stringKey("thread.name");
public static final AttributeKey<Long> THREAD_OS_ID = longKey("thread.os.id");
public static final AttributeKey<String> THREAD_STATE = stringKey("thread.state");
public static final AttributeKey<Boolean> THREAD_STACK_TRUNCATED =
booleanKey("thread.stack.truncated");

public static final AttributeKey<String> TRACE_ID = stringKey("trace_id");
public static final AttributeKey<String> SPAN_ID = stringKey("span_id");
Expand Down
20 changes: 18 additions & 2 deletions ...m/splunk/opentelemetry/profiler/allocation/exporter/PlainTextAllocationEventExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SCOPE_INFO;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;

import com.splunk.opentelemetry.profiler.Configuration.DataFormat;
import com.splunk.opentelemetry.profiler.LogDataCommonAttributes;
Expand All @@ -42,15 +43,18 @@ public class PlainTextAllocationEventExporter implements AllocationEventExporter
AttributeKey.longKey("memory.allocated");

private final StackSerializer stackSerializer;
private final int stackDepth;
private final LogProcessor logProcessor;
private final LogDataCommonAttributes commonAttributes;
private final Resource resource;

private PlainTextAllocationEventExporter(Builder builder) {
this.stackSerializer = builder.stackSerializer;
this.logProcessor = builder.logProcessor;
this.commonAttributes = builder.commonAttributes;
this.resource = builder.resource;
this.stackDepth = builder.stackDepth;
this.stackSerializer =
builder.stackSerializer != null ? builder.stackSerializer : new StackSerializer(stackDepth);
}

@Override
Expand All @@ -71,6 +75,12 @@ public void export(RecordedEvent event, AllocationEventSampler sampler, SpanCont
if (sampler != null) {
sampler.addAttributes((k, v) -> builder.put(k, v), (k, v) -> builder.put(k, v));
}

// stack trace is truncated either by jfr or in StackSerializer
if (stackTrace.isTruncated() || stackTrace.getFrames().size() > stackDepth) {
builder.put(THREAD_STACK_TRUNCATED, true);
}

Attributes attributes = builder.build();

LogDataBuilder logDataBuilder =
Expand Down Expand Up @@ -105,10 +115,11 @@ public static Builder builder() {
}

public static class Builder {
private StackSerializer stackSerializer = new StackSerializer();
private StackSerializer stackSerializer;
private LogProcessor logProcessor;
private LogDataCommonAttributes commonAttributes;
private Resource resource;
private int stackDepth;

public PlainTextAllocationEventExporter build() {
return new PlainTextAllocationEventExporter(this);
Expand All @@ -133,5 +144,10 @@ public Builder resource(Resource resource) {
this.resource = resource;
return this;
}

public Builder stackDepth(int stackDepth) {
this.stackDepth = stackDepth;
return this;
}
}
}
44 changes: 30 additions & 14 deletions ...a/com/splunk/opentelemetry/profiler/allocation/exporter/PprofAllocationEventExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_OS_ID;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID;

Expand All @@ -38,18 +39,19 @@
import io.opentelemetry.sdk.resources.Resource;
import java.time.Instant;
import jdk.jfr.consumer.RecordedEvent;
import jdk.jfr.consumer.RecordedFrame;
import jdk.jfr.consumer.RecordedMethod;
import jdk.jfr.consumer.RecordedStackTrace;
import jdk.jfr.consumer.RecordedThread;

public class PprofAllocationEventExporter implements AllocationEventExporter {
private final DataFormat dataFormat;
private final PprofLogDataExporter pprofLogDataExporter;
private final int stackDepth;
private Pprof pprof = createPprof();

private PprofAllocationEventExporter(Builder builder) {
this.dataFormat = builder.dataFormat;
this.stackDepth = builder.stackDepth;
this.pprofLogDataExporter =
new PprofLogDataExporter(
builder.logProcessor,
Expand All @@ -69,21 +71,29 @@ public void export(RecordedEvent event, AllocationEventSampler sampler, SpanCont

Sample.Builder sample = Sample.newBuilder();
sample.addValue(allocationSize);
// XXX StackSerializer limits stack depth. Although I believe jfr also limits it should we
// attempt to limit the depth here too?
for (RecordedFrame frame : stackTrace.getFrames()) {
RecordedMethod method = frame.getMethod();
if (method == null) {
sample.addLocationId(pprof.getLocationId("unknown", "unknown.unknown", -1));
} else {
sample.addLocationId(
pprof.getLocationId(
"unknown", // file name is not known
method.getType().getName() + "." + method.getName(),
frame.getLineNumber()));
}

if (stackTrace.isTruncated() || stackTrace.getFrames().size() > stackDepth) {
pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true);
}

stackTrace.getFrames().stream()
// limit the number of stack frames in case jfr stack depth is greater than our stack depth
// truncate the bottom stack frames the same way as jfr
.limit(stackDepth)
.forEachOrdered(
frame -> {
RecordedMethod method = frame.getMethod();
if (method == null) {
sample.addLocationId(pprof.getLocationId("unknown", "unknown.unknown", -1));
} else {
sample.addLocationId(
pprof.getLocationId(
"unknown", // file name is not known
method.getType().getName() + "." + method.getName(),
frame.getLineNumber()));
}
});

String eventName = event.getEventType().getName();
pprof.addLabel(sample, SOURCE_EVENT_NAME, eventName);
Instant time = event.getStartTime();
Expand Down Expand Up @@ -147,6 +157,7 @@ public static class Builder {
private LogProcessor logProcessor;
private Resource resource;
private DataFormat dataFormat;
private int stackDepth;

public PprofAllocationEventExporter build() {
return new PprofAllocationEventExporter(this);
Expand All @@ -166,5 +177,10 @@ public Builder dataFormat(DataFormat dataFormat) {
this.dataFormat = dataFormat;
return this;
}

public Builder stackDepth(int stackDepth) {
this.stackDepth = stackDepth;
return this;
}
}
}
15 changes: 14 additions & 1 deletion profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_OS_ID;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE;
import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID;

Expand All @@ -42,20 +43,22 @@
public class PprofCpuEventExporter implements CpuEventExporter {
private final DataFormat dataFormat;
private final EventPeriods eventPeriods;
private final int stackDepth;
private final PprofLogDataExporter pprofLogDataExporter;
private Pprof pprof = createPprof();

private PprofCpuEventExporter(Builder builder) {
this.dataFormat = builder.dataFormat;
this.eventPeriods = builder.eventPeriods;
this.stackDepth = builder.stackDepth;
this.pprofLogDataExporter =
new PprofLogDataExporter(
builder.logProcessor, builder.resource, ProfilingDataType.CPU, builder.dataFormat);
}

@Override
public void export(StackToSpanLinkage stackToSpanLinkage) {
StackTrace stackTrace = StackTraceParser.parse(stackToSpanLinkage.getRawStack());
StackTrace stackTrace = StackTraceParser.parse(stackToSpanLinkage.getRawStack(), stackDepth);
if (stackTrace == null || stackTrace.getStackTraceLines().isEmpty()) {
return;
}
Expand All @@ -71,6 +74,10 @@ public void export(StackToSpanLinkage stackToSpanLinkage) {
}
pprof.addLabel(sample, THREAD_STATE, stackTrace.getThreadState());

if (stackTrace.isTruncated()) {
pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true);
}

for (StackTraceParser.StackTraceLine stl : stackTrace.getStackTraceLines()) {
sample.addLocationId(
pprof.getLocationId(stl.getLocation(), stl.getClassAndMethod(), stl.getLineNumber()));
Expand Down Expand Up @@ -123,6 +130,7 @@ public static class Builder {
private Resource resource;
private DataFormat dataFormat;
private EventPeriods eventPeriods;
private int stackDepth;

public PprofCpuEventExporter build() {
return new PprofCpuEventExporter(this);
Expand All @@ -147,5 +155,10 @@ public Builder eventPeriods(EventPeriods eventPeriods) {
this.eventPeriods = eventPeriods;
return this;
}

public Builder stackDepth(int stackDepth) {
this.stackDepth = stackDepth;
return this;
}
}
}
Loading

0 comments on commit f685519

Please sign in to comment.