[logging] Format new platform json logging to ECS (#71138)

* [logging] Format new platform json logging to ECS * update integration tests * merge instead of assign * add @timestamp override test * add partial merge test against log object * add object level override test * fix type error Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
elastic · Jul 15, 2020 · 8bcecc0 · 8bcecc0
1 parent ed387dd
commit 8bcecc0
Show file tree

Hide file tree

Showing 6 changed files with 231 additions and 79 deletions.
diff --git a/src/core/server/logging/__snapshots__/logging_system.test.ts.snap b/src/core/server/logging/__snapshots__/logging_system.test.ts.snap
diff --git a/src/core/server/logging/integration_tests/logging.test.ts b/src/core/server/logging/integration_tests/logging.test.ts
@@ -198,13 +198,17 @@ describe('logging service', () => {
         JSON.parse(jsonString)
       );
       expect(firstCall).toMatchObject({
-        level: 'DEBUG',
-        context: 'plugins.myplugin.debug_json',
+        log: {
+          level: 'DEBUG',
+          logger: 'plugins.myplugin.debug_json',
+        },
         message: 'log1',
       });
       expect(secondCall).toMatchObject({
-        level: 'INFO',
-        context: 'plugins.myplugin.debug_json',
+        log: {
+          level: 'INFO',
+          logger: 'plugins.myplugin.debug_json',
+        },
         message: 'log2',
       });
     });
@@ -217,8 +221,10 @@ describe('logging service', () => {
 
       expect(mockConsoleLog).toHaveBeenCalledTimes(1);
       expect(JSON.parse(mockConsoleLog.mock.calls[0][0])).toMatchObject({
-        level: 'INFO',
-        context: 'plugins.myplugin.info_json',
+        log: {
+          level: 'INFO',
+          logger: 'plugins.myplugin.info_json',
+        },
         message: 'log2',
       });
     });
@@ -259,14 +265,18 @@ describe('logging service', () => {
       const logs = mockConsoleLog.mock.calls.map(([jsonString]) => jsonString);
 
       expect(JSON.parse(logs[0])).toMatchObject({
-        level: 'DEBUG',
-        context: 'plugins.myplugin.all',
+        log: {
+          level: 'DEBUG',
+          logger: 'plugins.myplugin.all',
+        },
         message: 'log1',
       });
       expect(logs[1]).toEqual('CUSTOM - PATTERN [plugins.myplugin.all][DEBUG] log1');
       expect(JSON.parse(logs[2])).toMatchObject({
-        level: 'INFO',
-        context: 'plugins.myplugin.all',
+        log: {
+          level: 'INFO',
+          logger: 'plugins.myplugin.all',
+        },
         message: 'log2',
       });
       expect(logs[3]).toEqual('CUSTOM - PATTERN [plugins.myplugin.all][INFO ] log2');

diff --git a/src/core/server/logging/layouts/__snapshots__/json_layout.test.ts.snap b/src/core/server/logging/layouts/__snapshots__/json_layout.test.ts.snap
diff --git a/src/core/server/logging/layouts/json_layout.test.ts b/src/core/server/logging/layouts/json_layout.test.ts
@@ -98,21 +98,27 @@ test('`format()` correctly formats record with meta-data', () => {
         timestamp,
         pid: 5355,
         meta: {
-          from: 'v7',
-          to: 'v8',
+          version: {
+            from: 'v7',
+            to: 'v8',
+          },
         },
       })
     )
   ).toStrictEqual({
     '@timestamp': '2012-02-01T09:30:22.011-05:00',
-    context: 'context-with-meta',
-    level: 'DEBUG',
+    log: {
+      level: 'DEBUG',
+      logger: 'context-with-meta',
+    },
     message: 'message-with-meta',
-    meta: {
+    version: {
       from: 'v7',
       to: 'v8',
     },
-    pid: 5355,
+    process: {
+      pid: 5355,
+    },
   });
 });
 
@@ -122,36 +128,131 @@ test('`format()` correctly formats error record with meta-data', () => {
   expect(
     JSON.parse(
       layout.format({
-        context: 'error-with-meta',
         level: LogLevel.Debug,
+        context: 'error-with-meta',
         error: {
           message: 'Some error message',
-          name: 'Some error name',
+          name: 'Some error type',
           stack: 'Some error stack',
         },
         message: 'Some error message',
         timestamp,
         pid: 5355,
         meta: {
-          from: 'v7',
-          to: 'v8',
+          version: {
+            from: 'v7',
+            to: 'v8',
+          },
         },
       })
     )
   ).toStrictEqual({
     '@timestamp': '2012-02-01T09:30:22.011-05:00',
-    context: 'error-with-meta',
-    level: 'DEBUG',
+    log: {
+      level: 'DEBUG',
+      logger: 'error-with-meta',
+    },
     error: {
       message: 'Some error message',
-      name: 'Some error name',
-      stack: 'Some error stack',
+      type: 'Some error type',
+      stack_trace: 'Some error stack',
     },
     message: 'Some error message',
-    meta: {
+    version: {
       from: 'v7',
       to: 'v8',
     },
-    pid: 5355,
+    process: {
+      pid: 5355,
+    },
+  });
+});
+
+test('format() meta can override @timestamp', () => {
+  const layout = new JsonLayout();
+  expect(
+    JSON.parse(
+      layout.format({
+        message: 'foo',
+        timestamp,
+        level: LogLevel.Debug,
+        context: 'bar',
+        pid: 3,
+        meta: {
+          '@timestamp': '2099-05-01T09:30:22.011-05:00',
+        },
+      })
+    )
+  ).toStrictEqual({
+    '@timestamp': '2099-05-01T09:30:22.011-05:00',
+    message: 'foo',
+    log: {
+      level: 'DEBUG',
+      logger: 'bar',
+    },
+    process: {
+      pid: 3,
+    },
+  });
+});
+
+test('format() meta can merge override logs', () => {
+  const layout = new JsonLayout();
+  expect(
+    JSON.parse(
+      layout.format({
+        timestamp,
+        message: 'foo',
+        level: LogLevel.Error,
+        context: 'bar',
+        pid: 3,
+        meta: {
+          log: {
+            kbn_custom_field: 'hello',
+          },
+        },
+      })
+    )
+  ).toStrictEqual({
+    '@timestamp': '2012-02-01T09:30:22.011-05:00',
+    message: 'foo',
+    log: {
+      level: 'ERROR',
+      logger: 'bar',
+      kbn_custom_field: 'hello',
+    },
+    process: {
+      pid: 3,
+    },
+  });
+});
+
+test('format() meta can override log level objects', () => {
+  const layout = new JsonLayout();
+  expect(
+    JSON.parse(
+      layout.format({
+        timestamp,
+        context: '123',
+        message: 'foo',
+        level: LogLevel.Error,
+        pid: 3,
+        meta: {
+          log: {
+            level: 'FATAL',
+          },
+        },
+      })
+    )
+  ).toStrictEqual({
+    '@timestamp': '2012-02-01T09:30:22.011-05:00',
+    message: 'foo',
+    log: {
+      level: 'FATAL',
+      logger: '123',
+    },
+    process: {
+      pid: 3,
+    },
   });
 });
diff --git a/src/core/server/logging/layouts/json_layout.ts b/src/core/server/logging/layouts/json_layout.ts
@@ -18,6 +18,7 @@
  */
 
 import moment from 'moment-timezone';
+import { merge } from 'lodash';
 import { schema, TypeOf } from '@kbn/config-schema';
 
 import { LogRecord } from '../log_record';
@@ -46,20 +47,28 @@ export class JsonLayout implements Layout {
 
     return {
       message: error.message,
-      name: error.name,
-      stack: error.stack,
+      type: error.name,
+      stack_trace: error.stack,
     };
   }
 
   public format(record: LogRecord): string {
-    return JSON.stringify({
-      '@timestamp': moment(record.timestamp).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
-      context: record.context,
-      error: JsonLayout.errorToSerializableObject(record.error),
-      level: record.level.id.toUpperCase(),
-      message: record.message,
-      meta: record.meta,
-      pid: record.pid,
-    });
+    return JSON.stringify(
+      merge(
+        {
+          '@timestamp': moment(record.timestamp).format('YYYY-MM-DDTHH:mm:ss.SSSZ'),
+          message: record.message,
+          error: JsonLayout.errorToSerializableObject(record.error),
+          log: {
+            level: record.level.id.toUpperCase(),
+            logger: record.context,
+          },
+          process: {
+            pid: record.pid,
+          },
+        },
+        record.meta
+      )
+    );
   }
 }