Telemetry processor - Allow multimatch on log body and span name

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/LogProcessor.java

@@ -3,7 +3,7 @@
 
 package com.microsoft.applicationinsights.agent.internal.processors;
 
-import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.applyRule;
+import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.applyRuleOnAllMatches;
 import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.getGroupNamesList;
 
 import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.ProcessorConfig;
@@ -104,7 +104,8 @@ public LogRecordData processToAttributes(LogRecordData log) {
     AttributesBuilder builder = log.getAttributes().toBuilder();
     for (int i = 0; i < groupNames.size(); i++) {
       bodyAsString =
-          applyRule(groupNames.get(i), toAttributeRulePatterns.get(i), bodyAsString, builder);
+          applyRuleOnAllMatches(
+              groupNames.get(i), toAttributeRulePatterns.get(i), bodyAsString, builder);
     }
 
     return new MyLogData(log, builder.build(), Body.string(bodyAsString));

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/ProcessorUtil.java

@@ -15,35 +15,68 @@
 
 public class ProcessorUtil {
 
-  public static String applyRule(
+  public static String applyRuleOnFirstMatch(
       List<String> groupNamesList, Pattern pattern, String name, AttributesBuilder builder) {
     if (groupNamesList.isEmpty()) {
       return name;
     }
     Matcher matcher = pattern.matcher(name);
     StringBuilder sb = new StringBuilder();
     int lastEnd = 0;
-    // As of now we are considering only first match.
     if (matcher.find()) {
-      sb.append(name, lastEnd, matcher.start());
-      int innerLastEnd = matcher.start();
-      for (int i = 1; i <= groupNamesList.size(); i++) {
-        sb.append(name, innerLastEnd, matcher.start(i));
-        sb.append("{");
-        sb.append(groupNamesList.get(i - 1));
-        // add attribute key=groupNames.get(i-1), value=matcher.group(i)
-        builder.put(groupNamesList.get(i - 1), matcher.group(i));
-        sb.append("}");
-        innerLastEnd = matcher.end(i);
-      }
-      sb.append(name, innerLastEnd, matcher.end());
-      lastEnd = matcher.end();
+      boolean firstMatch = true;
+      lastEnd = applyRule(groupNamesList, name, builder, sb, lastEnd, matcher, firstMatch);
+    }
+    sb.append(name, lastEnd, name.length());
+
+    return sb.toString();
+  }
+
+  // Rule applied on all matches for the returned String (not the attributes)
+  public static String applyRuleOnAllMatches(
+      List<String> groupNamesList, Pattern pattern, String name, AttributesBuilder builder) {
+    if (groupNamesList.isEmpty()) {
+      return name;
+    }
+    Matcher matcher = pattern.matcher(name);
+    StringBuilder sb = new StringBuilder();
+    int lastEnd = 0;
+    boolean firstMatch = true;
+    while (matcher.find()) {
+      lastEnd = applyRule(groupNamesList, name, builder, sb, lastEnd, matcher, firstMatch);
+      firstMatch = false;
     }
     sb.append(name, lastEnd, name.length());
 
     return sb.toString();
   }
 
+  private static int applyRule(
+      List<String> groupNamesList,
+      String name,
+      AttributesBuilder builder,
+      StringBuilder sb,
+      int lastEnd,
+      Matcher matcher,
+      boolean firstMatch) {
+    sb.append(name, lastEnd, matcher.start());
+    int innerLastEnd = matcher.start();
+    for (int i = 1; i <= groupNamesList.size(); i++) {
+      sb.append(name, innerLastEnd, matcher.start(i));
+      sb.append("{");
+      sb.append(groupNamesList.get(i - 1));
+      // add attribute key=groupNames.get(i-1), value=matcher.group(i)
+      if (firstMatch) {
+        builder.put(groupNamesList.get(i - 1), matcher.group(i));
+      }
+      sb.append("}");
+      innerLastEnd = matcher.end(i);
+    }
+    sb.append(name, innerLastEnd, matcher.end());
+    lastEnd = matcher.end();
+    return lastEnd;
+  }
+
   public static List<List<String>> getGroupNamesList(List<String> toAttributeRules) {
     List<List<String>> groupNamesList = new ArrayList<>();
     for (String rule : toAttributeRules) {

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/SpanProcessor.java

@@ -3,7 +3,7 @@
 
 package com.microsoft.applicationinsights.agent.internal.processors;
 
-import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.applyRule;
+import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.applyRuleOnFirstMatch;
 import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.getGroupNamesList;
 import static com.microsoft.applicationinsights.agent.internal.processors.ProcessorUtil.spanHasAllFromAttributeKeys;
 
@@ -101,7 +101,9 @@ public SpanData processToAttributes(SpanData span) {
     // they will be overwritten. Need a way to optimize this.
     AttributesBuilder builder = span.getAttributes().toBuilder();
     for (int i = 0; i < groupNames.size(); i++) {
-      spanName = applyRule(groupNames.get(i), toAttributeRulePatterns.get(i), spanName, builder);
+      spanName =
+          applyRuleOnFirstMatch(
+              groupNames.get(i), toAttributeRulePatterns.get(i), spanName, builder);
     }
     return new MySpanData(span, builder.build(), spanName);
   }

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/processors/ExporterWithLogProcessorTest.java

@@ -229,7 +229,7 @@ void multiRuleToAttributesTest() {
                 resultA.getAttributes().get(AttributeKey.stringKey("password2"))))
         .isEqualTo("555");
     assertThat(resultA.getBody().asString())
-        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass=777 zzz");
+        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass={password2} zzz");
     assertThat(
             Objects.requireNonNull(
                 resultB.getAttributes().get(AttributeKey.stringKey("password1"))))
@@ -241,6 +241,32 @@ void multiRuleToAttributesTest() {
     assertThat(resultB.getBody().asString()).isEqualTo("yyyPassword={password1} aba");
   }
 
+  @Test
+  void multiMatch() {
+    config.id = "MultiRuleToAttributes";
+    config.body = new NameConfig();
+    ToAttributeConfig toAttributeConfig = new ToAttributeConfig();
+    toAttributeConfig.rules = new ArrayList<>();
+    toAttributeConfig.rules.add("Password=(?<x>[^ ]+)");
+    config.body.toAttributes = toAttributeConfig;
+    LogRecordExporter logExporter = new ExporterWithLogProcessor(config, mockExporter);
+    TestLogRecordData mockLogA =
+        TestLogRecordData.builder()
+            .setBody("yyyPassword=123 aba Password=555 xyx")
+            .setAttributes(attributes)
+            .build();
+
+    List<LogRecordData> logs = new ArrayList<>();
+    logs.add(mockLogA);
+    logExporter.export(logs);
+
+    // verify that resulting logs are filtered in the way we want
+    List<LogRecordData> result = mockExporter.getLogs();
+    LogRecordData resultA = result.get(0);
+
+    assertThat(resultA.getBody().asString()).isEqualTo("yyyPassword={x} aba Password={x} xyx");
+  }
+
   @Test
   void simpleRenameLogTestWithLogProcessor() {
     config.id = "SimpleRenameLog";