Enhance Solr 7+ JMX metrics by including clustered Solr collections which include the collection name, shard name, and (replica) core

newrelic-agent/src/main/java/com/newrelic/agent/config/JmxConfig.java

@@ -30,4 +30,6 @@ public interface JmxConfig {
      * MBean server.
      */
     boolean registerLinkingMetadataMBean();
+
+    boolean enableIteratedObjectNameKeys();
 }

newrelic-agent/src/main/java/com/newrelic/agent/config/JmxConfigImpl.java

@@ -16,7 +16,9 @@ class JmxConfigImpl extends BaseConfig implements JmxConfig {
     public static final String ENABLED = "enabled";
     public static final String REGISTER_LINKING_METADATA_MBEAN = "linkingMetadataMBean";
     public static final String DISABLED_JMX_FRAMEWORKS = "disabled_jmx_frameworks";
+    public static final String ENABLE_ITERATED_OBJECTNAME_KEYS = "enable_iterated_objectname_keys";
     public static final boolean DEFAULT_REGISTER_LINKING_METADATA_MBEAN = false;
+    public static final boolean DEFAULT_ENABLE_ITERATED_OBJECTNAME_KEYS = true;
     public static final Boolean DEFAULT_ENABLED = Boolean.TRUE;
     public static final String SYSTEM_PROPERTY_ROOT = "newrelic.config.jmx.";
 
@@ -50,4 +52,9 @@ public Collection<String> getDisabledJmxFrameworks() {
     public boolean registerLinkingMetadataMBean(){
         return getProperty(REGISTER_LINKING_METADATA_MBEAN, DEFAULT_REGISTER_LINKING_METADATA_MBEAN);
     }
+
+    @Override
+    public boolean enableIteratedObjectNameKeys() {
+        return getProperty(ENABLE_ITERATED_OBJECTNAME_KEYS, DEFAULT_ENABLE_ITERATED_OBJECTNAME_KEYS);
+    }
 }

newrelic-agent/src/main/java/com/newrelic/agent/jmx/JmxApiImpl.java

@@ -17,6 +17,7 @@
 import com.newrelic.agent.jmx.values.JettyJmxMetrics;
 import com.newrelic.agent.jmx.values.KafkaConsumerJmxValues;
 import com.newrelic.agent.jmx.values.KafkaProducerJmxValues;
+import com.newrelic.agent.jmx.values.LegacySolr7JmxValues;
 import com.newrelic.agent.jmx.values.ResinJmxValues;
 import com.newrelic.agent.jmx.values.Solr7JmxValues;
 import com.newrelic.agent.jmx.values.SolrJmxValues;
@@ -55,6 +56,7 @@ public void addJmxMBeanGroup(String name) {
     }
 
     private JmxFrameworkValues getJmxFrameworkValues(String prefixName) {
+        JmxService jmxService = ServiceFactory.getJmxService();
         if (prefixName != null) {
             switch (prefixName) {
                 case KafkaProducerJmxValues.PREFIX:
@@ -68,7 +70,9 @@ private JmxFrameworkValues getJmxFrameworkValues(String prefixName) {
                 case SolrJmxValues.PREFIX:
                     return new SolrJmxValues();
                 case Solr7JmxValues.PREFIX:
-                    return new Solr7JmxValues();
+                    return jmxService.iteratedObjectNameKeysEnabled()
+                            ? new Solr7JmxValues()
+                            : new LegacySolr7JmxValues();
                 case WebsphereLibertyJmxValues.PREFIX:
                     return new WebsphereLibertyJmxValues();
                 case TomcatJmxValues.PREFIX:

newrelic-agent/src/main/java/com/newrelic/agent/jmx/JmxService.java

@@ -166,6 +166,10 @@ public void removeJmxServer(MBeanServer serverToRemove) {
         }
     }
 
+    public boolean iteratedObjectNameKeysEnabled() {
+        return jmxConfig.enableIteratedObjectNameKeys();
+    }
+
     private void process(StatsEngine statsEngine, Collection<MBeanServer> srvrList, JmxGet config) {
         ObjectName name = config.getObjectName();
         if (name == null) {

newrelic-agent/src/main/java/com/newrelic/agent/jmx/create/JmxGet.java

@@ -29,6 +29,7 @@
 import java.util.logging.Level;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.IntStream;
 
 /**
  * Used for attributes where we want to get their values.
@@ -38,6 +39,8 @@ public abstract class JmxGet extends JmxObject {
     private static final Pattern TYPE_QUERY_PATTERN = Pattern.compile(",(.*?)=");
     private static final Pattern PULL_VALUE_PATTERN = Pattern.compile("\\{(.*?)\\}");
     private static final Pattern PULL_ATTRIBUTE_PATTERN = Pattern.compile("\\:(.*?)\\:");
+    private static final Pattern PULL_ITER_VAL_PATTERN = Pattern.compile("for\\:(.*)\\[([0-9]+)\\:([0-9]*)\\:(.*)\\].*");
+    private static final Pattern RANGE_PATTERN = Pattern.compile("\\[([0-9]+)\\:([0-9]*)\\:(.*)\\]");
 
     /** This should be everything but the attribute portion of the metric. */
     private final String rootMetricName;
@@ -155,18 +158,21 @@ private String pullAttValuesFromName(ObjectName actualName, MBeanServer server)
         while (m.find()) {
 
             key = m.group(1);
-            Matcher attributeMatcher = PULL_ATTRIBUTE_PATTERN.matcher(key);
-            if (attributeMatcher.matches()) {
-                key = attributeMatcher.group(1);
-                try {
-                    value = server.getAttribute(actualName, key).toString();
-                } catch (Throwable e) {
-                    Agent.LOG.log(Level.FINEST, e, e.getMessage());
-                }
+
+            String iteratedValues = null;
+            try {
+                iteratedValues = matchAndGetIteratedValue(key, keyProperties, actualName, server);
+            }  catch (Throwable e) {
+                Agent.LOG.log(Level.FINEST, e, e.getMessage());
+            }
+
+            if (iteratedValues != null) {
+                value = iteratedValues;
             } else {
-                value = keyProperties.get(key);
+                value = getValueFromMBeanKey(key, keyProperties, actualName, server);
             }
 
+
             if (value != null) {
                 m.appendReplacement(sb, cleanValue(value));
             } else {
@@ -186,6 +192,68 @@ private String pullAttValuesFromName(ObjectName actualName, MBeanServer server)
         }
     }
 
+    private String matchAndGetIteratedValue(String key, Map<String, String> keyProperties,
+            ObjectName actualName, MBeanServer server) {
+
+        Matcher iterMatcher = PULL_ITER_VAL_PATTERN.matcher(key);
+        if (!iterMatcher.matches())  {
+            return null;
+        }
+
+        String rangedKeyFormat = key.substring(4);
+        Matcher rangeMatcher = RANGE_PATTERN.matcher(rangedKeyFormat);
+
+        if (!rangeMatcher.find()) {
+            return null;
+        }
+        String rangeStartStr = rangeMatcher.group(1);
+        String rangeEndStr = rangeMatcher.group(2);
+        String delimiterStr = rangeMatcher.group(3);
+        if (delimiterStr == null || delimiterStr.isEmpty()) {
+            delimiterStr = "/";
+        }
+        rangeMatcher.reset();
+
+        int rangeStart = (rangeStartStr != null && !rangeStartStr.isEmpty()) ? Integer.parseInt(rangeStartStr): 0;
+        int rangeEnd = (rangeEndStr != null && !rangeEndStr.isEmpty())
+                ? Integer.parseInt(rangeEndStr)
+                : rangeStart + keyProperties.size();
+
+        List<String> valueSequence = new ArrayList<>(keyProperties.size());
+
+        for (int i = rangeStart; i < rangeEnd && rangeMatcher.find(); i++) {
+            StringBuffer sb = new StringBuffer(key.length());
+            rangeMatcher.appendReplacement(sb, Integer.toString(i));
+            rangeMatcher.appendTail(sb);
+            rangeMatcher.reset();
+
+            String mbeanKey = sb.toString();
+            String value = getValueFromMBeanKey(mbeanKey, keyProperties, actualName, server);
+            if (value != null) {
+                valueSequence.add(value);
+            }
+        }
+
+        return String.join(delimiterStr, valueSequence);
+    }
+
+    private String getValueFromMBeanKey(String key, Map<String, String> keyProperties,
+            ObjectName actualName, MBeanServer server) {
+        String value = null;
+        Matcher pullAttrMatcher = PULL_ATTRIBUTE_PATTERN.matcher(key);
+        if (pullAttrMatcher.matches()) {
+            key = pullAttrMatcher.group(1);
+            try {
+                value = server.getAttribute(actualName, key).toString();
+            } catch (Throwable e) {
+                Agent.LOG.log(Level.FINEST, e, e.getMessage());
+            }
+        } else {
+            value = keyProperties.get(key);
+        }
+        return value;
+    }
+
     protected static String cleanValue(String value) {
         // remove leading slash
         value = value.trim();
@@ -263,4 +331,5 @@ protected JmxAttributeFilter getJmxAttributeFilter() {
     protected List<JmxMetric> getJmxMetrics() {
         return metrics;
     }
+
 }

newrelic-agent/src/main/java/com/newrelic/agent/jmx/metrics/BaseJmxValue.java

@@ -33,20 +33,124 @@ public class BaseJmxValue {
     private final JMXMetricType type;
     private final JmxMetricModifier modifier;
 
+    /**
+     * @param pObjectName
+     *  The object name query string used to search for JMX MBeans
+     * @param pObjectMetricName
+     *  A format string for agent metrics that matches against an MBean's object name.
+     *  An example format would be:
+     *
+     *  <blockquote><pre>
+     *  "TheRoot/Wahoo/{type}/{for:key[1::.]}/{otherKey}"
+     *  </pre></blockquote>
+     *
+     *  In the format we have placeholders such as:
+     *  <ul>
+     *      <li><b>{name}</b> which matches against a key in an object name and replaces it with a value.</li>
+     *      <li><b>{for:%[start:end:delimiter]%}</b> which matches against an iterated
+     *      sequence of keys starting from a non-negative number <i>start</i>
+     *      and ending it exclusively with <i>end</i> or unbounded if <i>end</i> is empty.
+     *      The placeholder is replaced with an iterated set of values each separated by the <i>delimiter</i> or if empty
+     *      "/" by default to follow the agent metric format.
+     *      For example: <b>{for:key[1::]}</b> and <b>{for:keyStart[1:3:.]keyEnd}</b>
+     *      </li>
+     *  </ul>
+     * @param pMetrics
+     */
     public BaseJmxValue(final String pObjectName, final String pObjectMetricName, JmxMetric[] pMetrics) {
         this(pObjectName, pObjectMetricName, null, null, JMXMetricType.INCREMENT_COUNT_PER_BEAN, pMetrics);
     }
 
+    /**
+     *
+     * @param pObjectName
+     *  The object name query string used to search for JMX MBeans
+     * @param pObjectMetricName
+     *  A format string for agent metrics that matches against an MBean's object name.
+     *  An example format would be:
+     *
+     *  <blockquote><pre>
+     *  "TheRoot/Wahoo/{type}/{for:key[1::.]}/{otherKey}"
+     *  </pre></blockquote>
+     *
+     *  In the format we have placeholders such as:
+     *  <ul>
+     *      <li><b>{name}</b> which matches against a key in an object name and replaces it with a value.</li>
+     *      <li><b>{for:%[start:end:delimiter]%}</b> which matches against an iterated
+     *      sequence of keys starting from a non-negative number <i>start</i>
+     *      and ending it exclusively with <i>end</i> or unbounded if <i>end</i> is empty.
+     *      The placeholder is replaced with an iterated set of values each separated by the <i>delimiter</i> or if empty
+     *      "/" by default to follow the agent metric format.
+     *      For example: <b>{for:key[1::]}</b> and <b>{for:keyStart[1:3:.]keyEnd}</b>
+     *      </li>
+     *  </ul>
+     * @param attributeFilter
+     * @param pMetrics
+     */
     public BaseJmxValue(final String pObjectName, final String pObjectMetricName, JmxAttributeFilter attributeFilter,
             JmxMetric[] pMetrics) {
         this(pObjectName, pObjectMetricName, attributeFilter, null, JMXMetricType.INCREMENT_COUNT_PER_BEAN, pMetrics);
     }
 
+    /**
+     *
+     * @param pObjectName
+     *  The object name query string used to search for JMX MBeans
+     * @param pObjectMetricName
+     *  A format string for agent metrics that matches against an MBean's object name.
+     *  An example format would be:
+     *
+     *  <blockquote><pre>
+     *  "TheRoot/Wahoo/{type}/{for:key[1::.]}/{otherKey}"
+     *  </pre></blockquote>
+     *
+     *  In the format we have placeholders such as:
+     *  <ul>
+     *      <li><b>{name}</b> which matches against a key in an object name and replaces it with a value.</li>
+     *      <li><b>{for:%[start:end:delimiter]%}</b> which matches against an iterated
+     *      sequence of keys starting from a non-negative number <i>start</i>
+     *      and ending it exclusively with <i>end</i> or unbounded if <i>end</i> is empty.
+     *      The placeholder is replaced with an iterated set of values each separated by the <i>delimiter</i> or if empty
+     *      "/" by default to follow the agent metric format.
+     *      For example: <b>{for:key[1::]}</b> and <b>{for:keyStart[1:3:.]keyEnd}</b>
+     *      </li>
+     *  </ul>
+     * @param pModifier
+     * @param pMetrics
+     */
     public BaseJmxValue(final String pObjectName, final String pObjectMetricName, JmxMetricModifier pModifier,
             JmxMetric[] pMetrics) {
         this(pObjectName, pObjectMetricName, null, pModifier, JMXMetricType.INCREMENT_COUNT_PER_BEAN, pMetrics);
     }
 
+    /**
+     *
+     * @param pObjectName
+     *  The object name query string used to search for JMX MBeans
+     * @param pObjectMetricName
+     *  A format string for agent metrics that matches against an MBean's object name.
+     *  An example format would be:
+     *
+     *  <blockquote><pre>
+     *  "TheRoot/Wahoo/{type}/{for:key[1::.]}/{otherKey}"
+     *  </pre></blockquote>
+     *
+     *  In the format we have placeholders such as:
+     *  <ul>
+     *      <li><b>{name}</b> which matches against a key in an object name and replaces it with a value.</li>
+     *      <li><b>{for:%[start:end:delimiter]%}</b> which matches against an iterated
+     *      sequence of keys starting from a non-negative number <i>start</i>
+     *      and ending it exclusively with <i>end</i> or unbounded if <i>end</i> is empty.
+     *      The placeholder is replaced with an iterated set of values each separated by the <i>delimiter</i> or if empty
+     *      "/" by default to follow the agent metric format.
+     *      For example: <b>{for:key[1::]}</b> and <b>{for:keyStart[1:3:.]keyEnd}</b>
+     *      </li>
+     *  </ul>
+     * @param attributeFilter
+     * @param pModifier
+     * @param pType
+     * @param pMetrics
+     */
     public BaseJmxValue(final String pObjectName, final String pObjectMetricName, JmxAttributeFilter attributeFilter,
             JmxMetricModifier pModifier, JMXMetricType pType, JmxMetric[] pMetrics) {
         objectNameString = pObjectName;