feat(core): add event and log streaming

* Added an event bus to Logger, which emits events when log entries are
created or updated.

* Added the `BufferedEventStream` class. This is used for batching and
streaming events from the Logger and the active Garden instance to the
platform when the user is logged in. Later, we can use this class for
streaming to the dashboard as well.

garden-service/src/analytics/analytics.ts

@@ -20,6 +20,7 @@ import { Events, EventName } from "../events"
 import { AnalyticsType } from "./analytics-types"
 import dedent from "dedent"
 import { getGitHubUrl } from "../docs/common"
+import { InternalError } from "../exceptions"
 
 const API_KEY = process.env.ANALYTICS_DEV ? SEGMENT_DEV_API_KEY : SEGMENT_PROD_API_KEY
 
@@ -59,7 +60,7 @@ export interface AnalyticsEventProperties {
   ciName: string | null
   system: SystemInfo
   isCI: boolean
-  sessionId: string
+  sessionId: string | null
   projectMetadata: ProjectMetadata
 }
 
@@ -134,14 +135,18 @@ export class AnalyticsHandler {
   private ciName = ci.name
   private systemConfig: SystemInfo
   private isCI = ci.isCI
-  private sessionId = uuidv4()
+  private sessionId: string
   protected garden: Garden
   private projectMetadata: ProjectMetadata
 
   private constructor(garden: Garden, log: LogEntry) {
+    if (!garden.sessionId) {
+      throw new InternalError(`Garden instance with null sessionId passed to AnalyticsHandler constructor.`, {})
+    }
     this.segment = new segmentClient(API_KEY, { flushAt: 20, flushInterval: 300 })
     this.log = log
     this.garden = garden
+    this.sessionId = garden.sessionId
     this.globalConfigStore = new GlobalConfigStore()
     this.analyticsConfig = {
       userId: "",

garden-service/src/cli/cli.ts

@@ -55,6 +55,7 @@ import { generateBasicDebugInfoReport } from "../commands/get/get-debug-info"
 import { AnalyticsHandler } from "../analytics/analytics"
 import { defaultDotIgnoreFiles } from "../util/fs"
 import { renderError } from "../logger/renderers"
+import { BufferedEventStream } from "../platform/buffered-event-stream"
 
 const OUTPUT_RENDERERS = {
   json: (data: DeepPrimitiveMap) => {
@@ -302,7 +303,9 @@ export class GardenCli {
       logger.info("")
       const footerLog = logger.placeholder()
 
+      // Init event & log streaming.
       const sessionId = uuidv4()
+      const bufferedEventStream = new BufferedEventStream(log, sessionId)
 
       const contextOpts: GardenOpts = {
         commandInfo: {
@@ -335,6 +338,11 @@ export class GardenCli {
           } else {
             garden = await Garden.factory(root, contextOpts)
           }
+
+          if (garden.clientAuthToken && garden.platformUrl) {
+            bufferedEventStream.connect(garden.events, garden.clientAuthToken, garden.platformUrl)
+          }
+
           // Register log file writers. We need to do this after the Garden class is initialised because
           // the file writers depend on the project root.
           await this.initFileWriters(logger, garden.projectRoot, garden.gardenDirPath)

garden-service/src/db/base-entity.ts

@@ -45,7 +45,6 @@ export class GardenEntity extends BaseEntity {
   /**
    * Helper method to avoid circular import issues.
    */
-
   static getConnection() {
     return getConnection()
   }

garden-service/src/events.ts

@@ -7,9 +7,9 @@
  */
 
 import { EventEmitter2 } from "eventemitter2"
-import { LogEntry } from "./logger/log-entry"
 import { ModuleVersion } from "./vcs/vcs"
 import { TaskResult } from "./task-graph"
+import { LogEntryEvent } from "./platform/buffered-event-stream"
 
 /**
  * This simple class serves as the central event bus for a Garden instance. Its function
@@ -18,7 +18,7 @@ import { TaskResult } from "./task-graph"
  * See below for the event interfaces.
  */
 export class EventBus extends EventEmitter2 {
-  constructor(private log: LogEntry) {
+  constructor() {
     super({
       wildcard: false,
       newListener: false,
@@ -27,7 +27,6 @@ export class EventBus extends EventEmitter2 {
   }
 
   emit<T extends EventName>(name: T, payload: Events[T]) {
-    this.log.silly(`Emit event '${name}'`)
     return super.emit(name, payload)
   }
 
@@ -47,9 +46,20 @@ export class EventBus extends EventEmitter2 {
 }
 
 /**
- * The supported events and their interfaces.
+ * Supported logger events and their interfaces.
  */
-export type Events = {
+export interface LoggerEvents {
+  _test: any
+  logEntryCreated: LogEntryEvent
+  logEntryUpdated: LogEntryEvent
+}
+
+export type LoggerEventName = keyof LoggerEvents
+
+/**
+ * Supported Garden events and their interfaces.
+ */
+export interface Events extends LoggerEvents {
   // Internal test/control events
   _exit: {}
   _restart: {}
@@ -108,8 +118,29 @@ export type Events = {
   taskGraphComplete: {
     completedAt: Date
   }
-
   watchingForChanges: {}
 }
 
 export type EventName = keyof Events
+
+// Note: Does not include logger events.
+export const eventNames: EventName[] = [
+  "_exit",
+  "_restart",
+  "_test",
+  "configAdded",
+  "configRemoved",
+  "internalError",
+  "projectConfigChanged",
+  "moduleConfigChanged",
+  "moduleSourcesChanged",
+  "moduleRemoved",
+  "taskPending",
+  "taskProcessing",
+  "taskComplete",
+  "taskError",
+  "taskCancelled",
+  "taskGraphProcessing",
+  "taskGraphComplete",
+  "watchingForChanges",
+]

garden-service/src/garden.ts

@@ -215,7 +215,7 @@ export class Garden {
     this.resolvedProviders = {}
 
     this.taskGraph = new TaskGraph(this, this.log)
-    this.events = new EventBus(this.log)
+    this.events = new EventBus()
 
     // Register plugins
     for (const plugin of [...builtinPlugins, ...params.plugins]) {
@@ -329,6 +329,7 @@ export class Garden {
    * Clean up before shutting down.
    */
   async close() {
+    this.events.removeAllListeners()
     this.watcher && (await this.watcher.stop())
   }
 

garden-service/src/logger/log-entry.ts

@@ -14,6 +14,7 @@ import { Omit } from "../util/util"
 import { getChildEntries, findParentEntry } from "./util"
 import { GardenError } from "../exceptions"
 import { Logger } from "./logger"
+import { formatForEventStream } from "../platform/buffered-event-stream"
 
 export type EmojiName = keyof typeof nodeEmoji.emoji
 export type LogSymbol = keyof typeof logSymbols | "empty"
@@ -33,6 +34,8 @@ export interface TaskMetadata {
   durationMs?: number
 }
 
+export const EVENT_LOG_LEVEL = LogLevel.debug
+
 interface MessageBase {
   msg?: string
   emoji?: EmojiName
@@ -90,6 +93,7 @@ export class LogEntry extends LogNode {
   public readonly childEntriesInheritLevel?: boolean
   public readonly id?: string
   public isPlaceholder?: boolean
+  public revision: number
 
   constructor(params: LogEntryConstructor) {
     super(params.level, params.parent, params.id)
@@ -102,6 +106,7 @@ export class LogEntry extends LogNode {
     this.metadata = params.metadata
     this.id = params.id
     this.isPlaceholder = params.isPlaceholder
+    this.revision = 0
 
     if (!params.isPlaceholder) {
       this.update({
@@ -114,6 +119,10 @@ export class LogEntry extends LogNode {
         dataFormat: params.dataFormat,
         maxSectionWidth: params.maxSectionWidth,
       })
+
+      if (this.level <= EVENT_LOG_LEVEL) {
+        this.root.events.emit("logEntryCreated", formatForEventStream(this))
+      }
     }
   }
 
@@ -124,6 +133,7 @@ export class LogEntry extends LogNode {
    * 3. next metadata is merged with the previous metadata
    */
   protected update(updateParams: UpdateLogEntryParams): void {
+    this.revision = this.revision + 1
     const messageState = this.getMessageState()
 
     // Explicitly set all the fields so the shape stays consistent
@@ -192,6 +202,9 @@ export class LogEntry extends LogNode {
 
   protected onGraphChange(node: LogEntry) {
     this.root.onGraphChange(node)
+    if (node.level <= EVENT_LOG_LEVEL) {
+      this.root.events.emit("logEntryUpdated", formatForEventStream(node))
+    }
   }
 
   getMetadata() {

garden-service/src/logger/logger.ts

@@ -17,6 +17,7 @@ import { FancyTerminalWriter } from "./writers/fancy-terminal-writer"
 import { JsonTerminalWriter } from "./writers/json-terminal-writer"
 import { parseLogLevel } from "../cli/helpers"
 import { FullscreenTerminalWriter } from "./writers/fullscreen-terminal-writer"
+import { EventBus } from "../events"
 
 export type LoggerType = "quiet" | "basic" | "fancy" | "fullscreen" | "json"
 export const LOGGER_TYPES = new Set<LoggerType>(["quiet", "basic", "fancy", "fullscreen", "json"])
@@ -44,6 +45,7 @@ export interface LoggerConfig {
 
 export class Logger extends LogNode {
   public writers: Writer[]
+  public events: EventBus
   public useEmoji: boolean
 
   private static instance: Logger
@@ -102,6 +104,7 @@ export class Logger extends LogNode {
     super(config.level)
     this.writers = config.writers || []
     this.useEmoji = config.useEmoji === false ? false : true
+    this.events = new EventBus()
   }
 
   protected createNode(params: CreateNodeParams): LogEntry {

garden-service/src/logger/writers/json-terminal-writer.ts

@@ -15,7 +15,6 @@ export interface JsonLogEntry {
   msg: string
   data?: any
   section?: string
-  durationMs?: number
   metadata?: LogEntryMetadata
 }