Skip to content

Commit

Permalink
[Profiling] Add TopN Functions API
Browse files Browse the repository at this point in the history 
With this commit we add a new API to the Universal Profiling plugin that
allows to gather a list of functions with the most observed samples
(TopN functions).
  • Loading branch information
@danielmitterdorfer
danielmitterdorfer committed Mar 28, 2024
1 parent 2a624e5 commit 35c2bcb
Show file tree
Hide file tree
Showing 22 changed files with 865 additions and 516 deletions.
16 changes: 15 additions & 1 deletion ...src/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetFlameGraphActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,21 @@

public class GetFlameGraphActionIT extends ProfilingTestCase {
public void testGetStackTracesUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
null,
null,
null,
null,
null,
null
);
GetFlamegraphResponse response = client().execute(GetFlamegraphAction.INSTANCE, request).get();
// only spot-check top level properties - detailed tests are done in unit tests
assertEquals(994, response.getSize());
Expand Down
91 changes: 90 additions & 1 deletion ...rc/internalClusterTest/java/org/elasticsearch/xpack/profiling/GetStackTracesActionIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,65 @@

public class GetStackTracesActionIT extends ProfilingTestCase {
public void testGetStackTracesUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(1000, 600.0d, 1.0d, 1.0d, null, null, null, null, null, null, null, null);
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
null,
null,
null,
null,
null,
null
);
request.setAdjustSampleCount(true);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
assertEquals(46, response.getTotalSamples());
assertEquals(1821, response.getTotalFrames());

assertNotNull(response.getStackTraceEvents());
assertEquals(3L, response.getStackTraceEvents().get("L7kj7UvlKbT-vN73el4faQ").count);

assertNotNull(response.getStackTraces());
// just do a high-level spot check. Decoding is tested in unit-tests
StackTrace stackTrace = response.getStackTraces().get("L7kj7UvlKbT-vN73el4faQ");
assertEquals(18, stackTrace.addressOrLines.length);
assertEquals(18, stackTrace.fileIds.length);
assertEquals(18, stackTrace.frameIds.length);
assertEquals(18, stackTrace.typeIds.length);
assertEquals(0.0000048475146d, stackTrace.annualCO2Tons, 0.0000000001d);
assertEquals(0.18834d, stackTrace.annualCostsUSD, 0.00001d);
// not determined by default
assertNull(stackTrace.subGroups);

assertNotNull(response.getStackFrames());
StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI");
assertEquals(List.of("start_thread"), stackFrame.functionName);

assertNotNull(response.getExecutables());
assertEquals("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
}

public void testGetStackTracesGroupedByServiceName() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
"service.name",
null,
null,
null,
null,
null
);
request.setAdjustSampleCount(true);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
assertEquals(46, response.getTotalSamples());
Expand All @@ -33,6 +91,7 @@ public void testGetStackTracesUnfiltered() throws Exception {
assertEquals(18, stackTrace.typeIds.length);
assertEquals(0.0000048475146d, stackTrace.annualCO2Tons, 0.0000000001d);
assertEquals(0.18834d, stackTrace.annualCostsUSD, 0.00001d);
assertEquals(Long.valueOf(2L), stackTrace.subGroups.get("basket"));

assertNotNull(response.getStackFrames());
StackFrame stackFrame = response.getStackFrames().get("8NlMClggx8jaziUTJXlmWAAAAAAAAIYI");
Expand All @@ -42,6 +101,28 @@ public void testGetStackTracesUnfiltered() throws Exception {
assertEquals("vmlinux", response.getExecutables().get("lHp5_WAgpLy2alrUVab6HA"));
}

public void testGetStackTracesGroupedByInvalidField() {
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
// only service.name is supported (note the trailing "s")
"service.names",
null,
null,
null,
null,
null
);
request.setAdjustSampleCount(true);
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, client().execute(GetStackTracesAction.INSTANCE, request));
assertEquals("Requested custom event aggregation field [service.names] but only [service.name] is supported.", e.getMessage());
}

public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception {
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must().add(QueryBuilders.termQuery("transaction.name", "encodeSha1"));
Expand All @@ -56,6 +137,7 @@ public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception
// also match an index that does not contain stacktrace ids to ensure it is ignored
new String[] { "apm-test-*", "apm-legacy-test-*" },
"transaction.profiler_stack_trace_ids",
"transaction.name",
null,
null,
null,
Expand All @@ -79,6 +161,7 @@ public void testGetStackTracesFromAPMWithMatchNoDownsampling() throws Exception
assertEquals(39, stackTrace.typeIds.length);
assertTrue(stackTrace.annualCO2Tons > 0.0d);
assertTrue(stackTrace.annualCostsUSD > 0.0d);
assertEquals(Long.valueOf(3L), stackTrace.subGroups.get("encodeSha1"));

assertNotNull(response.getStackFrames());
StackFrame stackFrame = response.getStackFrames().get("fhsEKXDuxJ-jIJrZpdRuSAAAAAAAAFtj");
Expand All @@ -103,6 +186,7 @@ public void testGetStackTracesFromAPMWithMatchAndDownsampling() throws Exception
null,
null,
null,
null,
null
);
// ensures consistent results in the random sampler aggregation that is used internally
Expand All @@ -126,6 +210,8 @@ public void testGetStackTracesFromAPMWithMatchAndDownsampling() throws Exception
assertEquals(39, stackTrace.typeIds.length);
assertTrue(stackTrace.annualCO2Tons > 0.0d);
assertTrue(stackTrace.annualCostsUSD > 0.0d);
// not determined by default
assertNull(stackTrace.subGroups);

assertNotNull(response.getStackFrames());
StackFrame stackFrame = response.getStackFrames().get("fhsEKXDuxJ-jIJrZpdRuSAAAAAAAAFtj");
Expand All @@ -150,6 +236,7 @@ public void testGetStackTracesFromAPMNoMatch() throws Exception {
null,
null,
null,
null,
null
);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
Expand All @@ -171,6 +258,7 @@ public void testGetStackTracesFromAPMIndexNotAvailable() throws Exception {
null,
null,
null,
null,
null
);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
Expand All @@ -192,6 +280,7 @@ public void testGetStackTracesFromAPMStackTraceFieldNotAvailable() throws Except
null,
null,
null,
null,
null
);
GetStackTracesResponse response = client().execute(GetStackTracesAction.INSTANCE, request).get();
Expand Down
84 changes: 84 additions & 0 deletions .../internalClusterTest/java/org/elasticsearch/xpack/profiling/GetTopNFunctionsActionIT.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.profiling;

import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;

public class GetTopNFunctionsActionIT extends ProfilingTestCase {
public void testGetTopNFunctionsUnfiltered() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
null,
null,
null,
null,
null,
null
);
request.setAdjustSampleCount(true);
GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
assertEquals(46, response.getTotalSamples());
assertEquals(747, response.getTopN().size());
}

public void testGetTopNFunctionsGroupedByServiceName() throws Exception {
GetStackTracesRequest request = new GetStackTracesRequest(
1000,
600.0d,
1.0d,
1.0d,
null,
null,
null,
"service.name",
null,
null,
null,
null,
null
);
request.setAdjustSampleCount(true);
request.setLimit(50);
GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
assertEquals(46, response.getTotalSamples());
assertEquals(50, response.getTopN().size());
}

public void testGetTopNFunctionsFromAPM() throws Exception {
BoolQueryBuilder query = QueryBuilders.boolQuery();
query.must().add(QueryBuilders.termQuery("transaction.name", "encodeSha1"));
query.must().add(QueryBuilders.rangeQuery("@timestamp").lte("1698624000"));

GetStackTracesRequest request = new GetStackTracesRequest(
null,
1.0d,
1.0d,
1.0d,
query,
// also match an index that does not contain stacktrace ids to ensure it is ignored
new String[] { "apm-test-*", "apm-legacy-test-*" },
"transaction.profiler_stack_trace_ids",
"transaction.name",
null,
null,
null,
null,
null
);
GetTopNFunctionsResponse response = client().execute(GetTopNFunctionsAction.INSTANCE, request).get();
assertEquals(5, response.getTotalSamples());
assertEquals(45, response.getTopN().size());
}
}
4 changes: 2 additions & 2 deletions x-pack/plugin/profiling/src/internalClusterTest/resources/data/profiling-events-all.ndjson
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@
{"create": {"_index": "profiling-events-all"}}
{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["XF9MchOwpePfa6_hYy-vZQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
{"create": {"_index": "profiling-events-all"}}
{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
{"Stacktrace.count": [2], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"], "service.name": "basket"}
{"create": {"_index": "profiling-events-all"}}
{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"]}
{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["L7kj7UvlKbT-vN73el4faQ"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["497295213074376"], "service.name": "basket"}
{"create": {"_index": "profiling-events-all"}}
{"Stacktrace.count": [1], "profiling.project.id": ["100"], "os.kernel": ["9.9.9-0"], "tags": ["environment:qa", "region:eu-west-1"], "host.ip": ["192.168.1.2"], "@timestamp": ["1698624000"], "container.name": ["instance-0000000010"], "ecs.version": ["1.12.0"], "Stacktrace.id": ["hRqQI2CBPiapzgFG9jrmDA"], "agent.version": ["head-be593ef3-1688111067"], "host.name": ["ip-192-168-1-2"], "host.id": ["8457605156473051743"], "process.thread.name": ["599103450330106"]}
{"create": {"_index": "profiling-events-all"}}
Expand Down
34 changes: 33 additions & 1 deletion ...ugin/profiling/src/main/java/org/elasticsearch/xpack/profiling/GetStackTracesRequest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,10 @@
public class GetStackTracesRequest extends ActionRequest implements IndicesRequest.Replaceable {
public static final ParseField QUERY_FIELD = new ParseField("query");
public static final ParseField SAMPLE_SIZE_FIELD = new ParseField("sample_size");
public static final ParseField LIMIT_FIELD = new ParseField("limit");
public static final ParseField INDICES_FIELD = new ParseField("indices");
public static final ParseField STACKTRACE_IDS_FIELD = new ParseField("stacktrace_ids_field");
public static final ParseField AGGREGATION_FIELD = new ParseField("aggregation_field");
public static final ParseField REQUESTED_DURATION_FIELD = new ParseField("requested_duration");
public static final ParseField AWS_COST_FACTOR_FIELD = new ParseField("aws_cost_factor");
public static final ParseField AZURE_COST_FACTOR_FIELD = new ParseField("azure_cost_factor");
Expand All @@ -52,9 +54,11 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque

private QueryBuilder query;
private int sampleSize;
private Integer limit;
private String[] indices;
private boolean userProvidedIndices;
private String stackTraceIdsField;
private String aggregationField;
private Double requestedDuration;
private Double awsCostFactor;
private Double azureCostFactor;
Expand All @@ -73,7 +77,7 @@ public class GetStackTracesRequest extends ActionRequest implements IndicesReque
private Integer shardSeed;

public GetStackTracesRequest() {
this(null, null, null, null, null, null, null, null, null, null, null, null);
this(null, null, null, null, null, null, null, null, null, null, null, null, null);
}

public GetStackTracesRequest(
Expand All @@ -84,6 +88,7 @@ public GetStackTracesRequest(
QueryBuilder query,
String[] indices,
String stackTraceIdsField,
String aggregationField,
Double customCO2PerKWH,
Double customDatacenterPUE,
Double customPerCoreWattX86,
Expand All @@ -98,6 +103,7 @@ public GetStackTracesRequest(
this.indices = indices;
this.userProvidedIndices = indices != null && indices.length > 0;
this.stackTraceIdsField = stackTraceIdsField;
this.aggregationField = aggregationField;
this.customCO2PerKWH = customCO2PerKWH;
this.customDatacenterPUE = customDatacenterPUE;
this.customPerCoreWattX86 = customPerCoreWattX86;
Expand All @@ -114,6 +120,14 @@ public int getSampleSize() {
return sampleSize;
}

public void setLimit(int limit) {
this.limit = limit;
}

public Integer getLimit() {
return limit;
}

public Double getRequestedDuration() {
return requestedDuration;
}
Expand Down Expand Up @@ -162,6 +176,10 @@ public String getStackTraceIdsField() {
return stackTraceIdsField;
}

public String getAggregationField() {
return aggregationField;
}

public boolean isAdjustSampleCount() {
return Boolean.TRUE.equals(adjustSampleCount);
}
Expand Down Expand Up @@ -194,8 +212,12 @@ public void parseXContent(XContentParser parser) throws IOException {
} else if (token.isValue()) {
if (SAMPLE_SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.sampleSize = parser.intValue();
} else if (LIMIT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.limit = parser.intValue();
} else if (STACKTRACE_IDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.stackTraceIdsField = parser.text();
} else if (AGGREGATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.aggregationField = parser.text();
} else if (REQUESTED_DURATION_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
this.requestedDuration = parser.doubleValue();
} else if (AWS_COST_FACTOR_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
Expand Down Expand Up @@ -277,7 +299,15 @@ public ActionRequestValidationException validate() {
);
}
}
if (aggregationField != null && aggregationField.isBlank()) {
validationException = addValidationError(
"[" + AGGREGATION_FIELD.getPreferredName() + "] must be non-empty",
validationException
);
}

validationException = requirePositive(SAMPLE_SIZE_FIELD, sampleSize, validationException);
validationException = requirePositive(LIMIT_FIELD, limit, validationException);
validationException = requirePositive(REQUESTED_DURATION_FIELD, requestedDuration, validationException);
validationException = requirePositive(AWS_COST_FACTOR_FIELD, awsCostFactor, validationException);
validationException = requirePositive(AZURE_COST_FACTOR_FIELD, azureCostFactor, validationException);
Expand Down Expand Up @@ -307,7 +337,9 @@ public String getDescription() {
StringBuilder sb = new StringBuilder();
appendField(sb, "indices", indices);
appendField(sb, "stacktrace_ids_field", stackTraceIdsField);
appendField(sb, "aggregation_field", aggregationField);
appendField(sb, "sample_size", sampleSize);
appendField(sb, "limit", limit);
appendField(sb, "requested_duration", requestedDuration);
appendField(sb, "aws_cost_factor", awsCostFactor);
appendField(sb, "azure_cost_factor", azureCostFactor);
Expand Down
Loading

0 comments on commit 35c2bcb

Please sign in to comment.