Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Features/dead lock detector #313

Merged
merged 2 commits into from
Aug 23, 2016
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
public class PerformanceCountersXmlElement {
private boolean useBuiltIn = true;
private long collectionFrequencyInSec = 60;

private PerformanceCounterJvmSectionXmlElement jvmSection;

private ArrayList<JmxXmlElement> jmxXmlElements;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,16 +21,17 @@

package com.microsoft.applicationinsights.internal.perfcounter;

import com.microsoft.applicationinsights.internal.logger.InternalLogger;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.DeadLockDetectorPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.GCPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.JvmMemoryPerformanceCounter;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;

import com.microsoft.applicationinsights.internal.logger.InternalLogger;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.DeadLockDetectorPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.JvmHeapMemoryUsedPerformanceCounter;

/**
* The class will create dedicated Jvm performance counters, unless disabled by user in the configuration file
*
* Created by gupele on 8/8/2016.
*/
public class JvmPerformanceCountersFactory implements PerformanceCountersFactory {
Expand All @@ -43,19 +44,22 @@ public Collection<PerformanceCounter> getPerformanceCounters() {
if (isEnabled) {
addDeadLockDetector(pcs);
addJvmMemoryPerformanceCounter(pcs);
addGCPerformanceCounter(pcs);
} else {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.TRACE, "JvmPerformanceCountersFactory is disabled");
}
return pcs;
}

private void addDeadLockDetector(ArrayList<PerformanceCounter> pcs) {
try {
if (disabledJvmPCs.contains(DeadLockDetectorPerformanceCounter.NAME)) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.TRACE, "DeadLockDetectorPerformanceCounter is disabled");
return;
}

DeadLockDetectorPerformanceCounter dlpc = new DeadLockDetectorPerformanceCounter();
if (!dlpc.isSupported()) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.TRACE, "DeadLockDetectorPerformanceCounter is not supported");
return;
}

Expand All @@ -67,27 +71,15 @@ private void addDeadLockDetector(ArrayList<PerformanceCounter> pcs) {

private void addJvmMemoryPerformanceCounter(ArrayList<PerformanceCounter> pcs) {
try {
if (disabledJvmPCs.contains(JvmMemoryPerformanceCounter.NAME)) {
return;
}

JvmMemoryPerformanceCounter mpc = new JvmMemoryPerformanceCounter();
pcs.add(mpc);
} catch (Throwable t) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.ERROR, "Failed to create JvmMemoryPerformanceCounter, exception: %s", t.getMessage());
}
}

private void addGCPerformanceCounter(ArrayList<PerformanceCounter> pcs) {
try {
if (disabledJvmPCs.contains(GCPerformanceCounter.NAME)) {
if (disabledJvmPCs.contains(JvmHeapMemoryUsedPerformanceCounter.NAME)) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.TRACE, "JvmHeapMemoryUsedPerformanceCounter is disabled");
return;
}

GCPerformanceCounter mpc = new GCPerformanceCounter();
JvmHeapMemoryUsedPerformanceCounter mpc = new JvmHeapMemoryUsedPerformanceCounter();
pcs.add(mpc);
} catch (Throwable t) {
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.ERROR, "Failed to create GCPerformanceCounter, exception: %s", t.getMessage());
InternalLogger.INSTANCE.logAlways(InternalLogger.LoggingLevel.ERROR, "Failed to create JvmHeapMemoryUsedPerformanceCounter, exception: %s", t.getMessage());
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@
import com.microsoft.applicationinsights.internal.config.PerformanceCountersXmlElement;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.DeadLockDetectorPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.GCPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.JvmMemoryPerformanceCounter;
import com.microsoft.applicationinsights.internal.perfcounter.jvm.JvmHeapMemoryUsedPerformanceCounter;

/**
* The class loads the relevant Jvm PCs
Expand Down Expand Up @@ -72,7 +72,7 @@ public final class JvmPerformanceCountersModule extends AbstractPerformanceCount

private String[] JvmPCNames = {
DeadLockDetectorPerformanceCounter.NAME,
JvmMemoryPerformanceCounter.NAME,
JvmHeapMemoryUsedPerformanceCounter.NAME,
GCPerformanceCounter.NAME
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,6 @@

package com.microsoft.applicationinsights.internal.perfcounter.jvm;

import com.microsoft.applicationinsights.TelemetryClient;
import com.microsoft.applicationinsights.internal.perfcounter.PerformanceCounter;
import com.microsoft.applicationinsights.telemetry.MetricTelemetry;

import java.lang.management.ManagementFactory;
import java.lang.management.MonitorInfo;
import java.lang.management.ThreadInfo;
Expand All @@ -33,11 +29,17 @@

import static java.lang.Math.min;

import com.microsoft.applicationinsights.TelemetryClient;
import com.microsoft.applicationinsights.internal.perfcounter.PerformanceCounter;
import com.microsoft.applicationinsights.internal.util.LocalStringsUtils;
import com.microsoft.applicationinsights.telemetry.MetricTelemetry;
import com.microsoft.applicationinsights.telemetry.TraceTelemetry;

/**
* The class uses the JVM ThreadMXBean to detect threads dead locks
* A metric with value 0 is sent when there are no blocked threads,
* otherwise the number of detected blocked threads is sent with a
* dimension that holds information like thread id and minimal stack traces
* dimension that holds information like thread id and minimal stack traces as trace telemetries
*
* Created by gupele on 8/7/2016.
*/
Expand All @@ -47,7 +49,7 @@ public final class DeadLockDetectorPerformanceCounter implements PerformanceCoun

private final static String INDENT = " ";
private final static String SEPERATOR = " | ";
private final static String METRIC_NAME = "Suspected deadlock Threads";
private final static String METRIC_NAME = "Suspected Deadlocked Threads";
private final static int MAX_STACK_TRACE = 3;

private final ThreadMXBean threadBean;
Expand Down Expand Up @@ -86,8 +88,14 @@ public void report(TelemetryClient telemetryClient) {
}

if (!blockedThreads.isEmpty()) {
String uuid = LocalStringsUtils.generateRandomIntegerId();

mt.setValue((double)blockedThreads.size());
mt.getContext().getProperties().putIfAbsent("Suspected threads that are in dead lock", sb.toString());
mt.getContext().getOperation().setId(uuid);

TraceTelemetry trace = new TraceTelemetry(String.format("%s%s", "Suspected deadlocked threads: ", sb.toString()));
trace.getContext().getOperation().setId(uuid);
telemetryClient.track(trace);
}
}
telemetryClient.track(mt);
Expand Down Expand Up @@ -118,7 +126,7 @@ private void setThreadInfo(StringBuilder sb, ThreadInfo ti) {
sb.append(ti.getThreadName());
sb.append(" Id=");
sb.append(ti.getThreadId());
sb.append(" in ");
sb.append(" is in ");
sb.append(ti.getThreadState());
if (ti.getLockName() != null) {
sb.append(" on lock=" + ti.getLockName());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,52 +30,44 @@
import com.microsoft.applicationinsights.telemetry.MetricTelemetry;

/**
* The class will create a metric telemetry for capturing the Jvm's heap memory usage
*
* Created by gupele on 8/8/2016.
*/
public class JvmMemoryPerformanceCounter implements PerformanceCounter {
public class JvmHeapMemoryUsedPerformanceCounter implements PerformanceCounter {

public final static String NAME = "MemoryUsage";

private final static String HEAP_MEM_USED = "Jvm Heap Memory Used";
private final static String HEAP_MEM_COMMITTED = "Jvm Heap Memory Committed";
private final static String HEAP_MEM_USED = "Heap Memory Used (MB)";

private final long Megabyte = 1024 * 1024;

private final static String NON_HEAP_MEM_USED = "Jvm Non Heap Memory Used";
private final static String NON_HEAP_MEM_COMMITTED = "Jvm Non Heap Memory Committed";
private final MemoryMXBean memory;

public JvmHeapMemoryUsedPerformanceCounter() {
memory = ManagementFactory.getMemoryMXBean();
}

@Override
public String getId() {
return "JvmHeapMemoryPerformanceCounter";
return "JvmHeapMemoryUsedPerformanceCounter";
}

@Override
public void report(TelemetryClient telemetryClient) {
MemoryMXBean memory = ManagementFactory.getMemoryMXBean();
if (memory == null) {
return;
}
reportHeap(memory, telemetryClient);
reportNonHeap(memory, telemetryClient);

}

private void reportNonHeap(MemoryMXBean memory, TelemetryClient telemetryClient) {
MemoryUsage mnhu = memory.getNonHeapMemoryUsage();
if (mnhu == null) {
return;
}
MetricTelemetry nonMemoryHeapUsage = new MetricTelemetry(NON_HEAP_MEM_USED, mnhu.getUsed());
MetricTelemetry nonMemoryHeapCommitted = new MetricTelemetry(NON_HEAP_MEM_COMMITTED, mnhu.getCommitted());

telemetryClient.track(nonMemoryHeapUsage);
telemetryClient.track(nonMemoryHeapCommitted);
reportHeap(memory, telemetryClient);
}

private void reportHeap(MemoryMXBean memory, TelemetryClient telemetryClient) {
MemoryUsage mhu = memory.getHeapMemoryUsage();
if (mhu != null) {
MetricTelemetry memoryHeapUsage = new MetricTelemetry(HEAP_MEM_USED, mhu.getUsed());
MetricTelemetry memoryHeapCommitted = new MetricTelemetry(HEAP_MEM_COMMITTED, mhu.getCommitted());
long currentHeapUsed = mhu.getUsed() / Megabyte;
MetricTelemetry memoryHeapUsage = new MetricTelemetry(HEAP_MEM_USED, currentHeapUsed);
telemetryClient.track(memoryHeapUsage);
telemetryClient.track(memoryHeapCommitted);
}
}
}