ingest: translate known OpenTelemetry JVM metrics (bp #4986) (#5047)

* ingest: translate known OpenTelemetry JVM metrics (#4986) * ingest: translate known OpenTelemetry JVM metrics Copy well-known JVM metrics from their OpenTelemetry names and associated labels to their counterparts produced by the Elastic APM Java agent. This enables users to visualise the metrics in the Elastic APM app in Kibana. Metrics are duplicated to avoid surprising users by silently dropping the OpenTelemetry names. Users can drop either one of the duplicates with ingest pipeline customisation. * Add changelog * docs: document new ingest pipelines * docs: move existing docs to correct position * ingest: address review comments - Move `runtime.jvm.gc.collection` before `runtime.jvm.gc.time` - Require `ctx.runtime?.jvm?.gc != null` when copying `labels.gc` * Regenerate integration package (cherry picked from commit 264d151) # Conflicts: # changelogs/head.asciidoc * Delete head.asciidoc Co-authored-by: Andrew Wilkins <axw@elastic.co> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
elastic · Apr 1, 2021 · d3cfeda · d3cfeda
1 parent e5ac1d1
commit d3cfeda
Show file tree

Hide file tree

Showing 14 changed files with 406 additions and 16 deletions.
diff --git a/....2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json b/....2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json
@@ -0,0 +1,41 @@
+{
+  "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+  "processors": [
+    {
+      "set": {
+        "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+        "copy_from": "runtime.jvm.memory.area",
+        "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.collection",
+        "if": "ctx.runtime?.jvm?.gc?.collection != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.time",
+        "if": "ctx.runtime?.jvm?.gc?.time != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.count",
+        "copy_from": "runtime.jvm.gc.count",
+        "if": "ctx.runtime?.jvm?.gc?.count != null"
+      }
+    },
+    {
+      "set": {
+        "field": "labels.name",
+        "copy_from": "labels.gc",
+        "override": false,
+        "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/app_metrics/elasticsearch/ingest_pipeline/default.json
@@ -26,6 +26,12 @@
         "name": "metrics-apm.app-0.2.0-apm_error_grouping_name",
         "if": "ctx.processor?.event == 'error'"
       }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.app-0.2.0-apm_opentelemetry_metrics",
+        "if": "ctx.processor?.event == 'metric'"
+      }
     }
   ]
 }
diff --git a/...0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json b/...0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json
@@ -0,0 +1,41 @@
+{
+  "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+  "processors": [
+    {
+      "set": {
+        "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+        "copy_from": "runtime.jvm.memory.area",
+        "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.collection",
+        "if": "ctx.runtime?.jvm?.gc?.collection != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.time",
+        "if": "ctx.runtime?.jvm?.gc?.time != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.count",
+        "copy_from": "runtime.jvm.gc.count",
+        "if": "ctx.runtime?.jvm?.gc?.count != null"
+      }
+    },
+    {
+      "set": {
+        "field": "labels.name",
+        "copy_from": "labels.gc",
+        "override": false,
+        "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/error_logs/elasticsearch/ingest_pipeline/default.json
@@ -26,6 +26,12 @@
         "name": "logs-apm.error-0.2.0-apm_error_grouping_name",
         "if": "ctx.processor?.event == 'error'"
       }
+    },
+    {
+      "pipeline": {
+        "name": "logs-apm.error-0.2.0-apm_opentelemetry_metrics",
+        "if": "ctx.processor?.event == 'metric'"
+      }
     }
   ]
 }
diff --git a/...data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json b/...data_stream/internal_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json
@@ -0,0 +1,41 @@
+{
+  "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+  "processors": [
+    {
+      "set": {
+        "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+        "copy_from": "runtime.jvm.memory.area",
+        "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.collection",
+        "if": "ctx.runtime?.jvm?.gc?.collection != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.time",
+        "if": "ctx.runtime?.jvm?.gc?.time != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.count",
+        "copy_from": "runtime.jvm.gc.count",
+        "if": "ctx.runtime?.jvm?.gc?.count != null"
+      }
+    },
+    {
+      "set": {
+        "field": "labels.name",
+        "copy_from": "labels.gc",
+        "override": false,
+        "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/internal_metrics/elasticsearch/ingest_pipeline/default.json
@@ -26,6 +26,12 @@
         "name": "metrics-apm.internal-0.2.0-apm_error_grouping_name",
         "if": "ctx.processor?.event == 'error'"
       }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.internal-0.2.0-apm_opentelemetry_metrics",
+        "if": "ctx.processor?.event == 'metric'"
+      }
     }
   ]
 }
diff --git a/.../data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json b/.../data_stream/profile_metrics/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json
@@ -0,0 +1,41 @@
+{
+  "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+  "processors": [
+    {
+      "set": {
+        "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+        "copy_from": "runtime.jvm.memory.area",
+        "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.collection",
+        "if": "ctx.runtime?.jvm?.gc?.collection != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.time",
+        "if": "ctx.runtime?.jvm?.gc?.time != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.count",
+        "copy_from": "runtime.jvm.gc.count",
+        "if": "ctx.runtime?.jvm?.gc?.count != null"
+      }
+    },
+    {
+      "set": {
+        "field": "labels.name",
+        "copy_from": "labels.gc",
+        "override": false,
+        "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/profile_metrics/elasticsearch/ingest_pipeline/default.json
@@ -26,6 +26,12 @@
         "name": "metrics-apm.profiling-0.2.0-apm_error_grouping_name",
         "if": "ctx.processor?.event == 'error'"
       }
+    },
+    {
+      "pipeline": {
+        "name": "metrics-apm.profiling-0.2.0-apm_opentelemetry_metrics",
+        "if": "ctx.processor?.event == 'metric'"
+      }
     }
   ]
 }
diff --git a/...apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json b/...apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/apm_opentelemetry_metrics.json
@@ -0,0 +1,41 @@
+{
+  "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+  "processors": [
+    {
+      "set": {
+        "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+        "copy_from": "runtime.jvm.memory.area",
+        "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.collection",
+        "if": "ctx.runtime?.jvm?.gc?.collection != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.time",
+        "copy_from": "runtime.jvm.gc.time",
+        "if": "ctx.runtime?.jvm?.gc?.time != null"
+      }
+    },
+    {
+      "set": {
+        "field": "jvm.gc.count",
+        "copy_from": "runtime.jvm.gc.count",
+        "if": "ctx.runtime?.jvm?.gc?.count != null"
+      }
+    },
+    {
+      "set": {
+        "field": "labels.name",
+        "copy_from": "labels.gc",
+        "override": false,
+        "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+      }
+    }
+  ]
+}
diff --git a/apmpackage/apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json b/apmpackage/apm/0.2.0/data_stream/traces/elasticsearch/ingest_pipeline/default.json
@@ -26,6 +26,12 @@
         "name": "traces-apm-0.2.0-apm_error_grouping_name",
         "if": "ctx.processor?.event == 'error'"
       }
+    },
+    {
+      "pipeline": {
+        "name": "traces-apm-0.2.0-apm_opentelemetry_metrics",
+        "if": "ctx.processor?.event == 'metric'"
+      }
     }
   ]
 }
diff --git a/docs/configuring-ingest.asciidoc b/docs/configuring-ingest.asciidoc
@@ -35,6 +35,17 @@ that are already available on the parent transaction.
 In previous versions of APM Server, this functionality was hardcoded internally.
 Switching metadata cleanup from an internal process to a processor allows you to keep any span metadata that is important in your architecture.
 
+`apm_error_grouping_name`::
+added:[7.13]
+Adds `error.grouping_name` to error documents for use in the {kibana-ref}/xpack-apm.html[Kibana APM UI].
+
+`apm_opentelemetry_metrics`::
+added:[7.13]
+Copies well-known OpenTelemetry metrics to their Elastic APM counterparts, for vizualisation in the {kibana-ref}/xpack-apm.html[Kibana APM UI].
+For example, the OpenTelemetry metric field `runtime.jvm.gc.time` is copied to the Elastic APM metric field `jvm.gc.time`.
+
+Metrics are duplicated so you can refer to them by either the OpenTelemetry or Elastic APM metric name.
+
 See the complete pipeline definition by navigating to the APM Server's home directory,
 and then viewing `ingest/pipeline/definition.json`.
 

diff --git a/ingest/pipeline/definition.json b/ingest/pipeline/definition.json
@@ -29,6 +29,12 @@
             "name": "apm_error_grouping_name",
             "if": "ctx.processor?.event == 'error'"
           }
+        },
+        {
+          "pipeline": {
+            "name": "apm_opentelemetry_metrics",
+            "if": "ctx.processor?.event == 'metric'"
+          }
         }
       ]
     }
@@ -137,5 +143,49 @@
         }
       ]
     }
+  },
+  {
+    "id": "apm_opentelemetry_metrics",
+    "body": {
+      "description": "Populate Elastic APM metric fields from well-known OpenTelemetry metric counterparts",
+      "processors": [
+        {
+          "set": {
+            "field": "jvm.memory.{{labels.area}}.{{labels.type}}",
+            "copy_from": "runtime.jvm.memory.area",
+            "if": "ctx.runtime?.jvm?.memory?.area != null && ctx.labels?.area != null && ctx.labels?.type != null"
+          }
+        },
+        {
+          "set": {
+            "field": "jvm.gc.time",
+            "copy_from": "runtime.jvm.gc.collection",
+            "if": "ctx.runtime?.jvm?.gc?.collection != null"
+          }
+        },
+        {
+          "set": {
+            "field": "jvm.gc.time",
+            "copy_from": "runtime.jvm.gc.time",
+            "if": "ctx.runtime?.jvm?.gc?.time != null"
+          }
+        },
+        {
+          "set": {
+            "field": "jvm.gc.count",
+            "copy_from": "runtime.jvm.gc.count",
+            "if": "ctx.runtime?.jvm?.gc?.count != null"
+          }
+        },
+        {
+          "set": {
+            "field": "labels.name",
+            "copy_from": "labels.gc",
+            "override": false,
+            "if": "ctx.labels?.gc != null && ctx.runtime?.jvm?.gc != null"
+          }
+        }
+      ]
+    }
   }
 ]