[ML] APM modules configs for RUM Javascript and NodeJS (#53792)

* [ML] apm modules

* [ML] apm modules

* [ML] update mocha test

* [ML] fix config

* [ML] single line JSON formatting for queries

* [ML] remove an empty path component with a trailing slash

* [ML] change detector descriptions, remove scroll size

* [ML] remove chunking_config from datafeeds

* [ML] fix configs

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.test.ts

@@ -345,6 +345,98 @@ describe('ML - custom URL utils', () => {
       );
     });
 
+    test('returns expected URL for APM', () => {
+      const urlConfig = {
+        url_name: 'APM',
+        time_range: '2h',
+        url_value:
+          'apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:"$trace.id$" and transaction.name:"$transaction.name$"&_g=()',
+      };
+
+      const testRecords = {
+        job_id: 'abnormal_trace_durations_nodejs',
+        result_type: 'record',
+        probability: 0.025597710862701226,
+        multi_bucket_impact: 5,
+        record_score: 13.124152090331723,
+        initial_record_score: 13.124152090331723,
+        bucket_span: 900,
+        detector_index: 0,
+        is_interim: false,
+        timestamp: 1573339500000,
+        by_field_name: 'transaction.name',
+        by_field_value: 'GET /test-data',
+        function: 'high_mean',
+        function_description: 'mean',
+        typical: [802.0600710562369],
+        actual: [761.1531339031332],
+        field_name: 'transaction.duration.us',
+        influencers: [
+          {
+            influencer_field_name: 'transaction.name',
+            influencer_field_values: ['GET /test-data'],
+          },
+          {
+            influencer_field_name: 'trace.id',
+            influencer_field_values: [
+              '000a09d58a428f38550e7e87637733c1',
+              '0039c771d8bbadf6137767d3aeb89f96',
+              '01279ed5bb9f4249e3822d16dec7f2f2',
+            ],
+          },
+          {
+            influencer_field_name: 'service.name',
+            influencer_field_values: ['example-service'],
+          },
+        ],
+        'trace.id': [
+          '000a09d58a428f38550e7e87637733c1',
+          '0039c771d8bbadf6137767d3aeb89f96',
+          '01279ed5bb9f4249e3822d16dec7f2f2',
+        ],
+        'service.name': ['example-service'],
+        'transaction.name': ['GET /test-data'],
+        earliest: '2019-11-09T20:45:00.000Z',
+        latest: '2019-11-10T01:00:00.000Z',
+      };
+
+      expect(getUrlForRecord(urlConfig, testRecords)).toBe(
+        'apm#/traces?rangeFrom=2019-11-09T20:45:00.000Z&rangeTo=2019-11-10T01:00:00.000Z&kuery=(trace.id:"000a09d58a428f38550e7e87637733c1" OR trace.id:"0039c771d8bbadf6137767d3aeb89f96" OR trace.id:"01279ed5bb9f4249e3822d16dec7f2f2") AND transaction.name:"GET%20%2Ftest-data"&_g=()'
+      );
+    });
+
+    test('removes an empty path component with a trailing slash', () => {
+      const urlConfig = {
+        url_name: 'APM',
+        time_range: '2h',
+        url_value:
+          'apm#/services/$service.name$/transactions?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request',
+      };
+
+      const testRecords = {
+        job_id: 'decreased_throughput_jsbase',
+        result_type: 'record',
+        probability: 8.91350850732573e-9,
+        multi_bucket_impact: 5,
+        record_score: 93.63625728951217,
+        initial_record_score: 93.63625728951217,
+        bucket_span: 900,
+        detector_index: 0,
+        is_interim: false,
+        timestamp: 1573266600000,
+        function: 'low_count',
+        function_description: 'count',
+        typical: [100615.66506877479],
+        actual: [25251],
+        earliest: '2019-11-09T00:30:00.000Z',
+        latest: '2019-11-09T04:45:00.000Z',
+      };
+
+      expect(getUrlForRecord(urlConfig, testRecords)).toBe(
+        'apm#/services/transactions?rangeFrom=2019-11-09T00:30:00.000Z&rangeTo=2019-11-09T04:45:00.000Z&refreshPaused=true&refreshInterval=0&kuery=&transactionType=request'
+      );
+    });
+
     test('returns expected URL for other type URL', () => {
       expect(getUrlForRecord(TEST_OTHER_URL, TEST_RECORD)).toBe(
         'http://airlinecodes.info/airline-code-AAL'

x-pack/legacy/plugins/ml/public/application/util/custom_url_utils.ts

@@ -97,7 +97,11 @@ export function openCustomUrlWindow(fullUrl: string, urlConfig: UrlConfig) {
 // a Kibana Discover or Dashboard page running on the same server as this ML plugin.
 function isKibanaUrl(urlConfig: UrlConfig) {
   const urlValue = urlConfig.url_value;
-  return urlValue.startsWith('kibana#/discover') || urlValue.startsWith('kibana#/dashboard');
+  return (
+    urlValue.startsWith('kibana#/discover') ||
+    urlValue.startsWith('kibana#/dashboard') ||
+    urlValue.startsWith('apm#/')
+  );
 }
 
 /**
@@ -136,13 +140,14 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
       commonEscapeCallback
     );
 
-    return str.replace(/\$([^?&$\'"]+)\$/g, (match, name: string) => {
+    // Looking for a $token$ with an optional trailing slash
+    return str.replace(/\$([^?&$\'"]+)\$(\/)?/g, (match, name: string, slash: string = '') => {
       // Use lodash get to allow nested JSON fields to be retrieved.
       let tokenValue: string | string[] | undefined = get(record, name);
       tokenValue = Array.isArray(tokenValue) ? tokenValue[0] : tokenValue;
 
-      // If property not found string is not replaced.
-      return tokenValue === undefined ? match : getResultTokenValue(tokenValue);
+      // If property not found token is replaced with an empty string.
+      return tokenValue === undefined ? '' : getResultTokenValue(tokenValue) + slash;
     });
   };
 
@@ -155,7 +160,7 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
         commonEscapeCallback
       );
       return str.replace(
-        /(.+query:')([^']*)('.+)/,
+        /(.+query:'|.+&kuery=)([^']*)(['&].+)/,
         (fullMatch, prefix: string, queryString: string, postfix: string) => {
           const [resultPrefix, resultPostfix] = [prefix, postfix].map(replaceSingleTokenValues);
 
@@ -170,28 +175,39 @@ function buildKibanaUrl(urlConfig: UrlConfig, record: CustomUrlAnomalyRecordDoc)
           const queryParts: string[] = [];
           const joinOperator = ' AND ';
 
-          for (let i = 0; i < queryFields.length; i++) {
+          fieldsLoop: for (let i = 0; i < queryFields.length; i++) {
             const field = queryFields[i];
             // Use lodash get to allow nested JSON fields to be retrieved.
-            const tokenValues: string[] | string | null = get(record, field) || null;
+            let tokenValues: string[] | string | null = get(record, field) || null;
             if (tokenValues === null) {
               continue;
             }
+            tokenValues = Array.isArray(tokenValues) ? tokenValues : [tokenValues];
+
             // Create a pair `influencerField:value`.
             // In cases where there are multiple influencer field values for an anomaly
             // combine values with OR operator e.g. `(influencerField:value or influencerField:another_value)`.
-            let result = (Array.isArray(tokenValues) ? tokenValues : [tokenValues])
-              .map(value => `${field}:"${getResultTokenValue(value)}"`)
-              .join(' OR ');
-            result = tokenValues.length > 1 ? `(${result})` : result;
-
-            // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
-            availableCharactersLeft -= result.length - (i === 0 ? 0 : joinOperator.length);
-
-            if (availableCharactersLeft <= 0) {
-              break;
-            } else {
-              queryParts.push(result);
+            let result = '';
+            for (let j = 0; j < tokenValues.length; j++) {
+              const part = `${j > 0 ? ' OR ' : ''}${field}:"${getResultTokenValue(
+                tokenValues[j]
+              )}"`;
+
+              // Build up a URL string which is not longer than the allowed length and isn't corrupted by invalid query.
+              if (availableCharactersLeft < part.length) {
+                if (result.length > 0) {
+                  queryParts.push(j > 0 ? `(${result})` : result);
+                }
+                break fieldsLoop;
+              }
+
+              result += part;
+
+              availableCharactersLeft -= result.length;
+            }
+
+            if (result.length > 0) {
+              queryParts.push(tokenValues.length > 1 ? `(${result})` : result);
             }
           }
 

x-pack/legacy/plugins/ml/server/models/data_recognizer/__tests__/data_recognizer.js

@@ -12,6 +12,8 @@ describe('ML - data recognizer', () => {
 
   const moduleIds = [
     'apache_ecs',
+    'apm_jsbase',
+    'apm_nodejs',
     'apm_transaction',
     'auditbeat_process_docker_ecs',
     'auditbeat_process_hosts_ecs',

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/logo.json

@@ -0,0 +1,3 @@
+{
+  "icon": "apmApp"
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/manifest.json

@@ -0,0 +1,53 @@
+{
+  "id": "apm_jsbase",
+  "title": "APM: RUM Javascript",
+  "description": "Detect problematic spans and identify user agents that are potentially causing issues.",
+  "type": "APM data",
+  "logoFile": "logo.json",
+  "defaultIndexPattern": "apm-*",
+  "query": {
+    "bool": {
+      "filter": [{ "term": { "agent.name": "js-base" } }]
+    }
+  },
+  "jobs": [
+    {
+      "id": "abnormal_span_durations_jsbase",
+      "file": "abnormal_span_durations_jsbase.json"
+    },
+    {
+      "id": "anomalous_error_rate_for_user_agents_jsbase",
+      "file": "anomalous_error_rate_for_user_agents_jsbase.json"
+    },
+    {
+      "id": "decreased_throughput_jsbase",
+      "file": "decreased_throughput_jsbase.json"
+    },
+    {
+      "id": "high_count_by_user_agent_jsbase",
+      "file": "high_count_by_user_agent_jsbase.json"
+    }
+  ],
+  "datafeeds": [
+    {
+      "id": "datafeed-abnormal_span_durations_jsbase",
+      "file": "datafeed_abnormal_span_durations_jsbase.json",
+      "job_id": "abnormal_span_durations_jsbase"
+    },
+    {
+      "id": "datafeed-anomalous_error_rate_for_user_agents_jsbase",
+      "file": "datafeed_anomalous_error_rate_for_user_agents_jsbase.json",
+      "job_id": "anomalous_error_rate_for_user_agents_jsbase"
+    },
+    {
+      "id": "datafeed-decreased_throughput_jsbase",
+      "file": "datafeed_decreased_throughput_jsbase.json",
+      "job_id": "decreased_throughput_jsbase"
+    },
+    {
+      "id": "datafeed-high_count_by_user_agent_jsbase",
+      "file": "datafeed_high_count_by_user_agent_jsbase.json",
+      "job_id": "high_count_by_user_agent_jsbase"
+    }
+  ]
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/abnormal_span_durations_jsbase.json

@@ -0,0 +1,41 @@
+{
+  "job_type": "anomaly_detector",
+  "groups": [
+    "apm"
+  ],
+  "description": "APM JSBase: Looks for spans that are taking longer than usual to process.",
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "increased span duration",
+        "function": "high_mean",
+        "field_name": "span.duration.us",
+        "partition_field_name": "span.type"
+      }
+    ],
+    "influencers": [
+      "span.type",
+      "trace.id",
+      "span.name",
+      "service.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "128mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-apm-jsbase",
+    "custom_urls": [
+      {
+        "url_name": "APM",
+        "time_range": "2h",
+        "url_value": "apm#/traces?rangeFrom=$earliest$&rangeTo=$latest$&kuery=trace.id:\"$trace.id$\"&_g=()"
+      }
+    ]
+  }
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/anomalous_error_rate_for_user_agents_jsbase.json

@@ -0,0 +1,40 @@
+{
+  "job_type": "anomaly_detector",
+  "groups": [
+    "apm"
+  ],
+  "description": "APM JSBase: Detects user agents that are encountering errors at an above normal rate. This can help detect browser compatibility issues.",
+  "analysis_config": {
+    "bucket_span": "15m",
+    "detectors": [
+      {
+        "detector_description": "high error rate for user agent",
+        "function": "high_non_zero_count",
+        "partition_field_name": "user_agent.name"
+      }
+    ],
+    "influencers": [
+      "user_agent.name",
+      "error.exception.message.keyword",
+      "error.page.url",
+      "service.name"
+    ]
+  },
+  "allow_lazy_open": true,
+  "analysis_limits": {
+    "model_memory_limit": "32mb"
+  },
+  "data_description": {
+    "time_field": "@timestamp"
+  },
+  "custom_settings": {
+    "created_by": "ml-module-apm-jsbase",
+    "custom_urls": [
+      {
+        "url_name": "APM",
+        "time_range": "2h",
+        "url_value": "apm#/services/$service.name$/errors?rangeFrom=$earliest$&rangeTo=$latest$&refreshPaused=true&refreshInterval=0&kuery=user_agent.name:\"$user_agent.name$\"&_g=()"
+      }
+    ]
+  }
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_abnormal_span_durations_jsbase.json

@@ -0,0 +1,15 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "must": [
+        { "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
+        { "bool": { "filter": { "term": { "processor.event": "span" } } } }
+      ]
+    }
+  }
+}

x-pack/legacy/plugins/ml/server/models/data_recognizer/modules/apm_jsbase/ml/datafeed_anomalous_error_rate_for_user_agents_jsbase.json

@@ -0,0 +1,15 @@
+{
+  "job_id": "JOB_ID",
+  "indices": [
+    "INDEX_PATTERN_NAME"
+  ],
+  "max_empty_searches": 10,
+  "query": {
+    "bool": {
+      "must": [
+        { "bool": { "filter": { "term": { "agent.name": "js-base" } } } },
+        { "exists": { "field": "user_agent.name" } }
+      ]
+    }
+  }
+}