Add an option to limit stack depth

profiler/build.gradle.kts

@@ -30,6 +30,8 @@ dependencies {
   compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-extension-api")
   compileOnly("io.opentelemetry.instrumentation:opentelemetry-instrumentation-api")
   compileOnly("io.opentelemetry:opentelemetry-semconv")
+  // required to access InstrumentationHolder
+  compileOnly("io.opentelemetry.javaagent:opentelemetry-javaagent-bootstrap")
   implementation("io.opentelemetry:opentelemetry-sdk-logs")
   implementation("io.opentelemetry:opentelemetry-exporter-otlp-logs")
   implementation("com.google.protobuf:protobuf-java:$protobufVersion")

profiler/src/main/java/com/splunk/opentelemetry/profiler/Configuration.java

@@ -36,6 +36,7 @@ public class Configuration implements ConfigPropertySource {
   public static final Duration DEFAULT_CALL_STACK_INTERVAL = Duration.ofSeconds(10);
   public static final boolean DEFAULT_INCLUDE_INTERNAL_STACKS = false;
   public static final boolean DEFAULT_TRACING_STACKS_ONLY = false;
+  private static final int DEFAULT_STACK_DEPTH = 1024;
 
   public static final String CONFIG_KEY_ENABLE_PROFILER = PROFILER_ENABLED_PROPERTY;
   public static final String CONFIG_KEY_PROFILER_DIRECTORY = "splunk.profiler.directory";
@@ -46,8 +47,8 @@ public class Configuration implements ConfigPropertySource {
   public static final String CONFIG_KEY_MEMORY_ENABLED = PROFILER_MEMORY_ENABLED_PROPERTY;
   public static final String CONFIG_KEY_MEMORY_SAMPLER_INTERVAL =
       "splunk.profiler.memory.sampler.interval";
-  public static final String CONFIG_KEY_CPU_DATA_FORMAT = "splunk.profiler.cpu.data.format";
-  public static final String CONFIG_KEY_MEMORY_DATA_FORMAT = "splunk.profiler.memory.data.format";
+  private static final String CONFIG_KEY_CPU_DATA_FORMAT = "splunk.profiler.cpu.data.format";
+  private static final String CONFIG_KEY_MEMORY_DATA_FORMAT = "splunk.profiler.memory.data.format";
   public static final String CONFIG_KEY_TLAB_ENABLED = "splunk.profiler.tlab.enabled";
   public static final String CONFIG_KEY_CALL_STACK_INTERVAL = "splunk.profiler.call.stack.interval";
   public static final String CONFIG_KEY_DEPRECATED_THREADDUMP_PERIOD =
@@ -60,6 +61,7 @@ public class Configuration implements ConfigPropertySource {
   public static final String CONFIG_KEY_INCLUDE_INTERNAL_STACKS =
       "splunk.profiler.include.internal.stacks";
   public static final String CONFIG_KEY_TRACING_STACKS_ONLY = "splunk.profiler.tracing.stacks.only";
+  private static final String CONFIG_KEY_STACK_DEPTH = "splunk.profiler.max.stack.depth";
 
   @Override
   public Map<String, String> getProperties() {
@@ -109,6 +111,10 @@ public static boolean getTracingStacksOnly(Config config) {
     return config.getBoolean(CONFIG_KEY_TRACING_STACKS_ONLY, DEFAULT_TRACING_STACKS_ONLY);
   }
 
+  public static int getStackDepth(Config config) {
+    return config.getInt(CONFIG_KEY_STACK_DEPTH, DEFAULT_STACK_DEPTH);
+  }
+
   public static DataFormat getCpuDataFormat(Config config) {
     String value = config.getString(CONFIG_KEY_CPU_DATA_FORMAT, "text");
     return getDataFormat(value);

profiler/src/main/java/com/splunk/opentelemetry/profiler/JFR.java

@@ -16,15 +16,24 @@
 
 package com.splunk.opentelemetry.profiler;
 
+import io.opentelemetry.javaagent.bootstrap.InstrumentationHolder;
 import java.io.IOException;
+import java.lang.instrument.Instrumentation;
 import java.nio.file.Path;
+import java.util.Collections;
 import jdk.jfr.FlightRecorder;
 import jdk.jfr.Recording;
 import jdk.jfr.consumer.RecordedEvent;
 import jdk.jfr.consumer.RecordingFile;
+import jdk.jfr.internal.Options;
+import net.bytebuddy.dynamic.loading.ClassInjector;
+import net.bytebuddy.utility.JavaModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /** Abstraction around the JDK Flight Recorder subsystem. */
 class JFR {
+  private static final Logger logger = LoggerFactory.getLogger(JFR.class);
 
   public static final JFR instance = new JFR();
   private static final boolean jfrAvailable = checkJfr();
@@ -61,4 +70,40 @@ public RecordedEvent readEvent(RecordingFile file, Path path) {
       throw new JfrException("Error reading events from " + path, e);
     }
   }
+
+  public void setStackDepth(int stackDepth) {
+    try {
+      JfrAccess.init();
+      int currentDepth = Options.getStackDepth();
+      // we only increase stack depth
+      if (currentDepth < stackDepth) {
+        Options.setStackDepth(stackDepth);
+      }
+    } catch (Throwable throwable) {
+      logger.warn("Failed to set JFR stack depth to {}", stackDepth, throwable);
+    }
+  }
+
+  private static class JfrAccess {
+    static {
+      Instrumentation instrumentation = InstrumentationHolder.getInstrumentation();
+      if (instrumentation != null && JavaModule.isSupported()) {
+        // ensure that we have access to jdk.jfr.internal.Options.setStackDepth
+        JavaModule currentModule = JavaModule.ofType(JFR.class);
+        JavaModule flightRecorder = JavaModule.ofType(FlightRecorder.class);
+        if (flightRecorder != null && flightRecorder.isNamed() && currentModule != null) {
+          ClassInjector.UsingInstrumentation.redefineModule(
+              instrumentation,
+              flightRecorder,
+              Collections.emptySet(),
+              Collections.emptyMap(),
+              Collections.singletonMap("jdk.jfr.internal", Collections.singleton(currentModule)),
+              Collections.emptySet(),
+              Collections.emptyMap());
+        }
+      }
+    }
+
+    static void init() {}
+  }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/JfrActivator.java

@@ -90,6 +90,9 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
     Path outputDir = Paths.get(config.getString(CONFIG_KEY_PROFILER_DIRECTORY));
     RecordingFileNamingConvention namingConvention = new RecordingFileNamingConvention(outputDir);
 
+    int stackDepth = Configuration.getStackDepth(config);
+    JFR.instance.setStackDepth(stackDepth);
+
     RecordingEscapeHatch recordingEscapeHatch =
         RecordingEscapeHatch.builder()
             .namingConvention(namingConvention)
@@ -122,6 +125,7 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
               .resource(resource)
               .dataFormat(cpuDataFormat)
               .eventPeriods(periods)
+              .stackDepth(stackDepth)
               .build();
     }
 
@@ -137,13 +141,15 @@ private void activateJfrAndRunForever(Config config, Resource resource) {
               .logProcessor(BatchLogProcessorHolder.get(logsExporter))
               .commonAttributes(commonAttributes)
               .resource(resource)
+              .stackDepth(stackDepth)
               .build();
     } else {
       allocationEventExporter =
           PprofAllocationEventExporter.builder()
               .logProcessor(SimpleLogProcessor.create(logsExporter))
               .resource(resource)
               .dataFormat(allocationDataFormat)
+              .stackDepth(stackDepth)
               .build();
     }
 

profiler/src/main/java/com/splunk/opentelemetry/profiler/ProfilingSemanticAttributes.java

@@ -16,6 +16,7 @@
 
 package com.splunk.opentelemetry.profiler;
 
+import static io.opentelemetry.api.common.AttributeKey.booleanKey;
 import static io.opentelemetry.api.common.AttributeKey.longKey;
 import static io.opentelemetry.api.common.AttributeKey.stringKey;
 
@@ -42,6 +43,8 @@ public class ProfilingSemanticAttributes {
   public static final AttributeKey<String> THREAD_NAME = stringKey("thread.name");
   public static final AttributeKey<Long> THREAD_OS_ID = longKey("thread.os.id");
   public static final AttributeKey<String> THREAD_STATE = stringKey("thread.state");
+  public static final AttributeKey<Boolean> THREAD_STACK_TRUNCATED =
+      booleanKey("thread.stack.truncated");
 
   public static final AttributeKey<String> TRACE_ID = stringKey("trace_id");
   public static final AttributeKey<String> SPAN_ID = stringKey("span_id");

profiler/src/main/java/com/splunk/opentelemetry/profiler/allocation/exporter/PlainTextAllocationEventExporter.java

@@ -19,6 +19,7 @@
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_FORMAT;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.DATA_TYPE;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.INSTRUMENTATION_SCOPE_INFO;
+import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;
 
 import com.splunk.opentelemetry.profiler.Configuration.DataFormat;
 import com.splunk.opentelemetry.profiler.LogDataCommonAttributes;
@@ -42,15 +43,18 @@ public class PlainTextAllocationEventExporter implements AllocationEventExporter
       AttributeKey.longKey("memory.allocated");
 
   private final StackSerializer stackSerializer;
+  private final int stackDepth;
   private final LogProcessor logProcessor;
   private final LogDataCommonAttributes commonAttributes;
   private final Resource resource;
 
   private PlainTextAllocationEventExporter(Builder builder) {
-    this.stackSerializer = builder.stackSerializer;
     this.logProcessor = builder.logProcessor;
     this.commonAttributes = builder.commonAttributes;
     this.resource = builder.resource;
+    this.stackDepth = builder.stackDepth;
+    this.stackSerializer =
+        builder.stackSerializer != null ? builder.stackSerializer : new StackSerializer(stackDepth);
   }
 
   @Override
@@ -71,6 +75,12 @@ public void export(RecordedEvent event, AllocationEventSampler sampler, SpanCont
     if (sampler != null) {
       sampler.addAttributes((k, v) -> builder.put(k, v), (k, v) -> builder.put(k, v));
     }
+
+    // stack trace is truncated either by jfr or in StackSerializer
+    if (stackTrace.isTruncated() || stackTrace.getFrames().size() > stackDepth) {
+      builder.put(THREAD_STACK_TRUNCATED, true);
+    }
+
     Attributes attributes = builder.build();
 
     LogDataBuilder logDataBuilder =
@@ -105,10 +115,11 @@ public static Builder builder() {
   }
 
   public static class Builder {
-    private StackSerializer stackSerializer = new StackSerializer();
+    private StackSerializer stackSerializer;
     private LogProcessor logProcessor;
     private LogDataCommonAttributes commonAttributes;
     private Resource resource;
+    private int stackDepth;
 
     public PlainTextAllocationEventExporter build() {
       return new PlainTextAllocationEventExporter(this);
@@ -133,5 +144,10 @@ public Builder resource(Resource resource) {
       this.resource = resource;
       return this;
     }
+
+    public Builder stackDepth(int stackDepth) {
+      this.stackDepth = stackDepth;
+      return this;
+    }
   }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/allocation/exporter/PprofAllocationEventExporter.java

@@ -22,6 +22,7 @@
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_OS_ID;
+import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID;
 
@@ -38,18 +39,19 @@
 import io.opentelemetry.sdk.resources.Resource;
 import java.time.Instant;
 import jdk.jfr.consumer.RecordedEvent;
-import jdk.jfr.consumer.RecordedFrame;
 import jdk.jfr.consumer.RecordedMethod;
 import jdk.jfr.consumer.RecordedStackTrace;
 import jdk.jfr.consumer.RecordedThread;
 
 public class PprofAllocationEventExporter implements AllocationEventExporter {
   private final DataFormat dataFormat;
   private final PprofLogDataExporter pprofLogDataExporter;
+  private final int stackDepth;
   private Pprof pprof = createPprof();
 
   private PprofAllocationEventExporter(Builder builder) {
     this.dataFormat = builder.dataFormat;
+    this.stackDepth = builder.stackDepth;
     this.pprofLogDataExporter =
         new PprofLogDataExporter(
             builder.logProcessor,
@@ -69,21 +71,29 @@ public void export(RecordedEvent event, AllocationEventSampler sampler, SpanCont
 
     Sample.Builder sample = Sample.newBuilder();
     sample.addValue(allocationSize);
-    // XXX StackSerializer limits stack depth. Although I believe jfr also limits it should we
-    // attempt to limit the depth here too?
-    for (RecordedFrame frame : stackTrace.getFrames()) {
-      RecordedMethod method = frame.getMethod();
-      if (method == null) {
-        sample.addLocationId(pprof.getLocationId("unknown", "unknown.unknown", -1));
-      } else {
-        sample.addLocationId(
-            pprof.getLocationId(
-                "unknown", // file name is not known
-                method.getType().getName() + "." + method.getName(),
-                frame.getLineNumber()));
-      }
+
+    if (stackTrace.isTruncated() || stackTrace.getFrames().size() > stackDepth) {
+      pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true);
     }
 
+    stackTrace.getFrames().stream()
+        // limit the number of stack frames in case jfr stack depth is greater than our stack depth
+        // truncate the bottom stack frames the same way as jfr
+        .limit(stackDepth)
+        .forEachOrdered(
+            frame -> {
+              RecordedMethod method = frame.getMethod();
+              if (method == null) {
+                sample.addLocationId(pprof.getLocationId("unknown", "unknown.unknown", -1));
+              } else {
+                sample.addLocationId(
+                    pprof.getLocationId(
+                        "unknown", // file name is not known
+                        method.getType().getName() + "." + method.getName(),
+                        frame.getLineNumber()));
+              }
+            });
+
     String eventName = event.getEventType().getName();
     pprof.addLabel(sample, SOURCE_EVENT_NAME, eventName);
     Instant time = event.getStartTime();
@@ -145,6 +155,7 @@ public static class Builder {
     private LogProcessor logProcessor;
     private Resource resource;
     private DataFormat dataFormat;
+    private int stackDepth;
 
     public PprofAllocationEventExporter build() {
       return new PprofAllocationEventExporter(this);
@@ -164,5 +175,10 @@ public Builder dataFormat(DataFormat dataFormat) {
       this.dataFormat = dataFormat;
       return this;
     }
+
+    public Builder stackDepth(int stackDepth) {
+      this.stackDepth = stackDepth;
+      return this;
+    }
   }
 }

profiler/src/main/java/com/splunk/opentelemetry/profiler/exporter/PprofCpuEventExporter.java

@@ -23,6 +23,7 @@
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_ID;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_NAME;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_OS_ID;
+import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STACK_TRUNCATED;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.THREAD_STATE;
 import static com.splunk.opentelemetry.profiler.ProfilingSemanticAttributes.TRACE_ID;
 
@@ -42,20 +43,22 @@
 public class PprofCpuEventExporter implements CpuEventExporter {
   private final DataFormat dataFormat;
   private final EventPeriods eventPeriods;
+  private final int stackDepth;
   private final PprofLogDataExporter pprofLogDataExporter;
   private Pprof pprof = createPprof();
 
   private PprofCpuEventExporter(Builder builder) {
     this.dataFormat = builder.dataFormat;
     this.eventPeriods = builder.eventPeriods;
+    this.stackDepth = builder.stackDepth;
     this.pprofLogDataExporter =
         new PprofLogDataExporter(
             builder.logProcessor, builder.resource, ProfilingDataType.CPU, builder.dataFormat);
   }
 
   @Override
   public void export(StackToSpanLinkage stackToSpanLinkage) {
-    StackTrace stackTrace = StackTraceParser.parse(stackToSpanLinkage.getRawStack());
+    StackTrace stackTrace = StackTraceParser.parse(stackToSpanLinkage.getRawStack(), stackDepth);
     if (stackTrace == null || stackTrace.getStackTraceLines().isEmpty()) {
       return;
     }
@@ -69,6 +72,10 @@ public void export(StackToSpanLinkage stackToSpanLinkage) {
     pprof.addLabel(sample, THREAD_OS_ID, stackTrace.getOsThreadId());
     pprof.addLabel(sample, THREAD_STATE, stackTrace.getThreadState());
 
+    if (stackTrace.isTruncated()) {
+      pprof.addLabel(sample, THREAD_STACK_TRUNCATED, true);
+    }
+
     for (StackTraceParser.StackTraceLine stl : stackTrace.getStackTraceLines()) {
       sample.addLocationId(
           pprof.getLocationId(stl.getLocation(), stl.getClassAndMethod(), stl.getLineNumber()));
@@ -121,6 +128,7 @@ public static class Builder {
     private Resource resource;
     private DataFormat dataFormat;
     private EventPeriods eventPeriods;
+    private int stackDepth;
 
     public PprofCpuEventExporter build() {
       return new PprofCpuEventExporter(this);
@@ -145,5 +153,10 @@ public Builder eventPeriods(EventPeriods eventPeriods) {
       this.eventPeriods = eventPeriods;
       return this;
     }
+
+    public Builder stackDepth(int stackDepth) {
+      this.stackDepth = stackDepth;
+      return this;
+    }
   }
 }