✨ [RUMF-727] introduce v2 format (#570)

* ♻️ move types to a dedicated file

* ✨ define v2 types and events

* ✨ generate long tasks in v2

* ✨ handle v2 events

* ✨ add format validation during unit tests

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "rum-events-format"]
+	path = rum-events-format
+	url = https://github.com/DataDog/rum-events-format

.prettierignore

@@ -2,3 +2,4 @@ bundle
 cjs
 esm
 coverage
+rum-events-format

LICENSE-3rdparty.csv

@@ -13,6 +13,7 @@ dev,@wdio/local-runner,MIT,Copyright JS Foundation and other contributors
 dev,@wdio/selenium-standalone-service,MIT,Copyright JS Foundation and other contributors
 dev,@wdio/spec-reporter,MIT,Copyright JS Foundation and other contributors
 dev,@wdio/sync,MIT,Copyright JS Foundation and other contributors
+dev,ajv,MIT,Copyright 2015-2017 Evgeny Poberezkin
 dev,browserstack-local,MIT,Copyright 2016 BrowserStack
 dev,codecov,MIT,Copyright 2014 Gregg Caines
 dev,cors,MIT,Copyright 2013 Troy Goode

package.json

@@ -6,6 +6,7 @@
     "packages/*"
   ],
   "scripts": {
+    "postinstall": "git submodule update --init",
     "build": "lerna run build --stream",
     "build:bundle": "lerna run build:bundle --stream",
     "format": "prettier --check \"**/*.{ts,js,json,md,yml,html}\"",

packages/core/src/errorCollection.ts

@@ -18,7 +18,7 @@ export interface ErrorMessage {
 export interface ErrorContext {
   kind?: string
   stack?: string
-  origin: ErrorOrigin
+  origin: ErrorSource
 }
 
 export interface HttpContext {
@@ -27,7 +27,7 @@ export interface HttpContext {
   method: string
 }
 
-export enum ErrorOrigin {
+export enum ErrorSource {
   AGENT = 'agent',
   CONSOLE = 'console',
   NETWORK = 'network',
@@ -61,7 +61,7 @@ export function filterErrors(configuration: Configuration, errorObservable: Obse
       filteredErrorObservable.notify({
         context: {
           error: {
-            origin: ErrorOrigin.AGENT,
+            origin: ErrorSource.AGENT,
           },
         },
         message: `Reached max number of errors by minute: ${configuration.maxErrorsByMinute}`,
@@ -82,7 +82,7 @@ export function startConsoleTracking(errorObservable: ErrorObservable) {
     errorObservable.notify({
       context: {
         error: {
-          origin: ErrorOrigin.CONSOLE,
+          origin: ErrorSource.CONSOLE,
         },
       },
       message: ['console error:', message, ...optionalParams].map(formatConsoleParameters).join(' '),
@@ -134,7 +134,7 @@ export function formatRuntimeError(stackTrace: StackTrace, errorObject: any) {
       error: {
         stack,
         kind: stackTrace.name,
-        origin: ErrorOrigin.SOURCE,
+        origin: ErrorSource.SOURCE,
       },
     },
     startTime: performance.now(),
@@ -162,7 +162,7 @@ export function trackNetworkError(configuration: Configuration, errorObservable:
       errorObservable.notify({
         context: {
           error: {
-            origin: ErrorOrigin.NETWORK,
+            origin: ErrorSource.NETWORK,
             stack: truncateResponse(request.response, configuration) || 'Failed to load',
           },
           http: {

packages/core/src/index.ts

@@ -5,7 +5,7 @@ export {
   isIntakeRequest,
   buildCookieOptions,
 } from './configuration'
-export { ErrorMessage, ErrorContext, HttpContext, ErrorOrigin, ErrorObservable } from './errorCollection'
+export { ErrorMessage, ErrorContext, HttpContext, ErrorSource, ErrorObservable } from './errorCollection'
 export {
   BuildEnv,
   BuildMode,

packages/core/src/utils.ts

@@ -17,7 +17,7 @@ export enum DOM_EVENT {
   DOM_CONTENT_LOADED = 'DOMContentLoaded',
 }
 
-export enum ResourceKind {
+export enum ResourceType {
   DOCUMENT = 'document',
   XHR = 'xhr',
   BEACON = 'beacon',
@@ -31,8 +31,8 @@ export enum ResourceKind {
 }
 
 export enum RequestType {
-  FETCH = ResourceKind.FETCH,
-  XHR = ResourceKind.XHR,
+  FETCH = ResourceType.FETCH,
+  XHR = ResourceType.XHR,
 }
 
 // use lodash API

packages/core/test/errorCollection.spec.ts

@@ -2,7 +2,7 @@ import { FetchStub, FetchStubManager, isIE, SPEC_ENDPOINTS, stubFetch } from '..
 import { Configuration } from '../src/configuration'
 import {
   ErrorMessage,
-  ErrorOrigin,
+  ErrorSource,
   filterErrors,
   formatRuntimeError,
   startConsoleTracking,
@@ -21,7 +21,7 @@ describe('console tracker', () => {
   const CONSOLE_CONTEXT = {
     context: {
       error: {
-        origin: ErrorOrigin.CONSOLE,
+        origin: ErrorSource.CONSOLE,
       },
     },
   }
@@ -300,7 +300,7 @@ describe('error limitation', () => {
   const CONTEXT = {
     context: {
       error: {
-        origin: ErrorOrigin.SOURCE,
+        origin: ErrorSource.SOURCE,
       },
     },
     startTime: 100,
@@ -335,7 +335,7 @@ describe('error limitation', () => {
     errorObservable.notify({ message: '3', ...CONTEXT })
 
     expect(filteredSubscriber).toHaveBeenCalledWith({
-      context: { error: { origin: ErrorOrigin.AGENT } },
+      context: { error: { origin: ErrorSource.AGENT } },
       message: 'Reached max number of errors by minute: 2',
       startTime: jasmine.any(Number),
     })

packages/logs/src/logger.ts

@@ -1,4 +1,4 @@
-import { combine, Context, ContextValue, createContextManager, ErrorOrigin, monitored } from '@datadog/browser-core'
+import { combine, Context, ContextValue, createContextManager, ErrorSource, monitored } from '@datadog/browser-core'
 
 export enum StatusType {
   debug = 'debug',
@@ -75,7 +75,7 @@ export class Logger {
   error(message: string, messageContext?: Context) {
     const errorOrigin = {
       error: {
-        origin: ErrorOrigin.LOGGER,
+        origin: ErrorSource.LOGGER,
       },
     }
     this.log(message, combine(errorOrigin, messageContext), StatusType.error)

packages/logs/test/logs.spec.ts

@@ -4,7 +4,7 @@ import {
   DEFAULT_CONFIGURATION,
   ErrorMessage,
   ErrorObservable,
-  ErrorOrigin,
+  ErrorSource,
   noop,
   Observable,
 } from '@datadog/browser-core'
@@ -217,7 +217,7 @@ describe('logs', () => {
       startLogs({ errorLogger: new Logger(sendLogSpy) })
 
       errorObservable.notify({
-        context: { error: { origin: ErrorOrigin.SOURCE, kind: 'Error' } },
+        context: { error: { origin: ErrorSource.SOURCE, kind: 'Error' } },
         message: 'error!',
         startTime: 1234,
       })
@@ -226,7 +226,7 @@ describe('logs', () => {
       expect(sendLogSpy.calls.first().args).toEqual([
         {
           date: jasmine.any(Number),
-          error: { origin: ErrorOrigin.SOURCE, kind: 'Error' },
+          error: { origin: ErrorSource.SOURCE, kind: 'Error' },
           message: 'error!',
           status: StatusType.error,
         },
@@ -245,7 +245,7 @@ describe('logs', () => {
       startLogs({ errorLogger: new Logger(sendLogSpy) })
 
       errorObservable.notify({
-        context: { error: { origin: ErrorOrigin.SOURCE, kind: 'Error' } },
+        context: { error: { origin: ErrorSource.SOURCE, kind: 'Error' } },
         message: 'error!',
         startTime: 1234,
       })

packages/rum/package.json

@@ -18,6 +18,7 @@
   },
   "devDependencies": {
     "@types/sinon": "7.0.13",
+    "ajv": "6.12.6",
     "sinon": "7.3.2"
   },
   "repository": {

packages/rum/src/assembly.ts

@@ -1,37 +1,13 @@
 import { combine, Configuration, Context, withSnakeCaseKeys } from '@datadog/browser-core'
 import { LifeCycle, LifeCycleEventType } from './lifeCycle'
-import { ActionContext, ParentContexts, ViewContext } from './parentContexts'
-import {
-  RawRumEvent,
-  RumErrorEvent,
-  RumEventCategory,
-  RumLongTaskEvent,
-  RumResourceEvent,
-  RumUserActionEvent,
-  RumViewEvent,
-} from './rum'
+import { ParentContexts } from './parentContexts'
 import { RumSession } from './rumSession'
+import { RawRumEvent, RumContext, RumErrorEvent, RumEventCategory, RumLongTaskEvent, RumResourceEvent } from './types'
 
 interface BrowserWindow extends Window {
   _DATADOG_SYNTHETICS_BROWSER?: unknown
 }
 
-interface RumContext {
-  applicationId: string
-  date: number
-  service?: string
-  session: {
-    type: string
-  }
-}
-
-export type RumEvent =
-  | RumErrorEvent & ActionContext & ViewContext & RumContext
-  | RumResourceEvent & ActionContext & ViewContext & RumContext
-  | RumViewEvent & ViewContext & RumContext
-  | RumLongTaskEvent & ActionContext & ViewContext & RumContext
-  | RumUserActionEvent & ViewContext & RumContext
-
 enum SessionType {
   SYNTHETICS = 'synthetics',
   USER = 'user',

packages/rum/src/assemblyV2.ts

@@ -0,0 +1,85 @@
+import { combine, Configuration, Context, withSnakeCaseKeys } from '@datadog/browser-core'
+import { LifeCycle, LifeCycleEventType } from './lifeCycle'
+import { ParentContexts } from './parentContexts'
+import { RumSession } from './rumSession'
+import { RawRumEvent, RumContext, RumErrorEvent, RumEventCategory, RumLongTaskEvent, RumResourceEvent } from './types'
+import {
+  RawRumEventV2,
+  RumContextV2,
+  RumErrorEventV2,
+  RumEventType,
+  RumLongTaskEventV2,
+  RumResourceEventV2,
+} from './typesV2'
+
+interface BrowserWindow extends Window {
+  _DATADOG_SYNTHETICS_BROWSER?: unknown
+}
+
+enum SessionType {
+  SYNTHETICS = 'synthetics',
+  USER = 'user',
+}
+
+export function startRumAssemblyV2(
+  applicationId: string,
+  configuration: Configuration,
+  lifeCycle: LifeCycle,
+  session: RumSession,
+  parentContexts: ParentContexts,
+  getGlobalContext: () => Context
+) {
+  lifeCycle.subscribe(
+    LifeCycleEventType.RAW_RUM_EVENT_V2_COLLECTED,
+    ({
+      startTime,
+      rawRumEvent,
+      savedGlobalContext,
+      customerContext,
+    }: {
+      startTime: number
+      rawRumEvent: RawRumEventV2
+      savedGlobalContext?: Context
+      customerContext?: Context
+    }) => {
+      const viewContext = parentContexts.findViewV2(startTime)
+      if (session.isTracked() && viewContext && viewContext.session.id) {
+        const actionContext = parentContexts.findActionV2(startTime)
+        const rumContext: RumContextV2 = {
+          _dd: {
+            formatVersion: 2,
+          },
+          application: {
+            id: applicationId,
+          },
+          date: new Date().getTime(),
+          service: configuration.service,
+          session: {
+            // must be computed on each event because synthetics instrumentation can be done after sdk execution
+            // cf https://github.com/puppeteer/puppeteer/issues/3667
+            type: getSessionType(),
+          },
+        }
+        const rumEvent = needToAssembleWithAction(rawRumEvent)
+          ? combine(rumContext, viewContext, actionContext, rawRumEvent)
+          : combine(rumContext, viewContext, rawRumEvent)
+        const serverRumEvent = combine(
+          savedGlobalContext || getGlobalContext(),
+          customerContext,
+          withSnakeCaseKeys(rumEvent)
+        )
+        lifeCycle.notify(LifeCycleEventType.RUM_EVENT_V2_COLLECTED, { rumEvent, serverRumEvent })
+      }
+    }
+  )
+}
+
+function needToAssembleWithAction(
+  event: RawRumEventV2
+): event is RumErrorEventV2 | RumResourceEventV2 | RumLongTaskEventV2 {
+  return [RumEventType.ERROR, RumEventType.RESOURCE, RumEventType.LONG_TASK].indexOf(event.type) !== -1
+}
+
+function getSessionType() {
+  return (window as BrowserWindow)._DATADOG_SYNTHETICS_BROWSER === undefined ? SessionType.USER : SessionType.SYNTHETICS
+}

packages/rum/src/batch.ts

@@ -1,7 +1,7 @@
 import { Batch, combine, Configuration, Context, HttpRequest } from '@datadog/browser-core'
-import { RumEvent } from './assembly'
 import { LifeCycle, LifeCycleEventType } from './lifeCycle'
-import { RumEventCategory } from './rum'
+import { RumEvent, RumEventCategory } from './types'
+import { RumEventType, RumEventV2 } from './typesV2'
 
 export function startRumBatch(configuration: Configuration, lifeCycle: LifeCycle) {
   const batch = makeRumBatch(configuration, lifeCycle)
@@ -17,6 +17,17 @@ export function startRumBatch(configuration: Configuration, lifeCycle: LifeCycle
     }
   )
 
+  lifeCycle.subscribe(
+    LifeCycleEventType.RUM_EVENT_V2_COLLECTED,
+    ({ rumEvent, serverRumEvent }: { rumEvent: RumEventV2; serverRumEvent: Context }) => {
+      if (rumEvent.type === RumEventType.VIEW) {
+        batch.upsert(serverRumEvent, rumEvent.view.id)
+      } else {
+        batch.add(serverRumEvent)
+      }
+    }
+  )
+
   return {
     stop() {
       batch.stop()

packages/rum/src/index.ts

@@ -1,4 +1,10 @@
 export { Datacenter } from '@datadog/browser-core'
-export { RumUserConfiguration, RumGlobal, datadogRum, InternalContext } from './rum.entry'
-export { RumEventCategory, RumResourceEvent, RumViewEvent, RumUserActionEvent } from './rum'
-export { RumEvent } from './assembly'
+export { RumUserConfiguration, RumGlobal, datadogRum } from './rum.entry'
+export {
+  InternalContext,
+  RumEvent,
+  RumEventCategory,
+  RumUserActionEvent,
+  RumViewEvent,
+  RumResourceEvent,
+} from './types'