feat(logger): update dump strategy, fix koishijs#50

Author: shigma

plugins/logger/client/index.vue

@@ -2,8 +2,11 @@
   <k-layout>
     <el-scrollbar class="container">
       <div ref="root" class="logs">
-        <div class="line" :class="{ start: line.includes(hint) }" v-for="line in store.logs">
-          <code v-html="renderLine(line)"></code>
+        <div
+          v-for="(record, index) in store.logs"
+          :key="record.id"
+          class="line" :class="{ start: index && store.logs[index - 1].id > record.id && record.name === 'app' }">
+          <code v-html="renderLine(record)"></code>
         </div>
       </div>
     </el-scrollbar>
@@ -13,18 +16,50 @@
 <script lang="ts" setup>
 
 import { watch, ref, nextTick, onActivated } from 'vue'
-import { store } from '@koishijs/client'
+import type { Logger } from 'koishi'
+import { store, Time } from '@koishijs/client'
 import ansi from 'ansi_up'
 
 const root = ref<HTMLElement>()
 
-const hint = `app\u001b[0m \u001b[38;5;15;1mKoishi/`
-
 // this package does not have consistent exports in different environments
 const converter = new (ansi['default'] || ansi)()
 
-function renderLine(line: string) {
-  return converter.ansi_to_html(line)
+function renderColor(code: number, value: any, decoration = '') {
+  return `\u001b[3${code < 8 ? code : '8;5;' + code}${decoration}m${value}\u001b[0m`
+}
+
+function colorCode(name: string) {
+  let hash = 0
+  for (let i = 0; i < name.length; i++) {
+    hash = ((hash << 3) - hash) + name.charCodeAt(i)
+    hash |= 0
+  }
+  return c256[Math.abs(hash) % c256.length]
+}
+
+const showTime = 'yyyy-MM-dd hh:mm:ss'
+
+const c256 = [
+  20, 21, 26, 27, 32, 33, 38, 39, 40, 41, 42, 43, 44, 45, 56, 57, 62,
+  63, 68, 69, 74, 75, 76, 77, 78, 79, 80, 81, 92, 93, 98, 99, 112, 113,
+  129, 134, 135, 148, 149, 160, 161, 162, 163, 164, 165, 166, 167, 168,
+  169, 170, 171, 172, 173, 178, 179, 184, 185, 196, 197, 198, 199, 200,
+  201, 202, 203, 204, 205, 206, 207, 208, 209, 214, 215, 220, 221,
+]
+
+function renderLine(record: Logger.Record) {
+  const prefix = `[${record.type[0].toUpperCase()}]`
+  const space = ' '
+  let indent = 3 + space.length, output = ''
+  indent += showTime.length + space.length
+  output += renderColor(8, Time.template(showTime)) + space
+  const code = colorCode(record.name)
+  const label = renderColor(code, record.name, ';1')
+  const padLength = label.length - record.name.length
+  output += prefix + space + label.padEnd(padLength) + space
+  output += record.content.replace(/\n/g, '\n' + ' '.repeat(indent))
+  return converter.ansi_to_html(output)
 }
 
 onActivated(() => {

plugins/logger/src/file.ts

@@ -0,0 +1,55 @@
+import { FileHandle, open } from 'fs/promises'
+import { Logger } from 'koishi'
+
+export class FileWriter {
+  public data: Logger.Record[]
+  public task: Promise<FileHandle>
+  public size: number
+
+  private temp: Logger.Record[] = []
+
+  constructor(public date: string, public path: string) {
+    this.task = open(path, 'a+').then(async (handle) => {
+      const buffer = await handle.readFile()
+      this.data = this.parse(new TextDecoder().decode(buffer))
+      this.size = buffer.byteLength
+      return handle
+    })
+    this.task.then(() => this.flush())
+  }
+
+  flush() {
+    if (!this.temp.length) return
+    this.task = this.task.then(async (handle) => {
+      const content = Buffer.from(this.temp.map((record) => JSON.stringify(record) + '\n').join(''))
+      await handle.write(content)
+      this.data.push(...this.temp)
+      this.size += content.byteLength
+      this.temp = []
+      return handle
+    })
+  }
+
+  parse(text: string) {
+    return text.split('\n').map((line) => {
+      try {
+        return JSON.parse(line)
+      } catch {}
+    }).filter(Boolean)
+  }
+
+  async read() {
+    await this.task
+    return this.data
+  }
+
+  write(record: Logger.Record) {
+    this.temp.push(record)
+    this.flush()
+  }
+
+  async close() {
+    const handle = await this.task
+    await handle.close()
+  }
+}

plugins/logger/src/index.ts

@@ -1,10 +1,9 @@
-import { Context, Logger, remove, Schema, Time } from 'koishi'
+import { Context, Dict, Logger, remove, Schema, Time } from 'koishi'
 import { DataService } from '@koishijs/plugin-console'
 import { resolve } from 'path'
-import { promises as fsp, mkdirSync, readdirSync } from 'fs'
-import { FileHandle } from 'fs/promises'
-
-const { open, rm } = fsp
+import { mkdirSync, readdirSync } from 'fs'
+import { rm } from 'fs/promises'
+import { FileWriter } from './file'
 
 declare module '@koishijs/plugin-console' {
   namespace Console {
@@ -14,10 +13,10 @@ declare module '@koishijs/plugin-console' {
   }
 }
 
-class LogProvider extends DataService<string[]> {
+class LogProvider extends DataService<Logger.Record[]> {
   root: string
   date: string
-  files: number[] = []
+  files: Dict<number> = {}
   writer: FileWriter
 
   constructor(ctx: Context, private config: LogProvider.Config = {}) {
@@ -28,55 +27,57 @@ class LogProvider extends DataService<string[]> {
       prod: resolve(__dirname, '../dist'),
     })
 
-    this.ctx.on('ready', () => {
+    ctx.on('ready', () => {
       this.prepareWriter()
       this.prepareLogger()
     }, true)
+
+    ctx.on('dispose', () => {
+      this.writer?.close()
+      this.writer = null
+    })
   }
 
   prepareWriter() {
     this.root = resolve(this.ctx.baseDir, this.config.root)
     mkdirSync(this.root, { recursive: true })
 
     for (const filename of readdirSync(this.root)) {
-      if (!filename.endsWith('.log')) continue
-      this.files.push(Time.getDateNumber(new Date(filename.slice(0, -4)), 0))
+      const capture = /^(\d{4}-\d{2}-\d{2})-(\d+)\.log$/.exec(filename)
+      if (!capture) continue
+      this.files[capture[1]] = Math.max(this.files[capture[1]] ?? 0, +capture[2])
     }
 
-    this.createFile()
-
-    this.ctx.on('dispose', () => {
-      this.writer?.close()
-      this.writer = null
-    })
+    const date = new Date().toISOString().slice(0, 10)
+    this.createFile(date, this.files[date] ??= 1)
   }
 
-  createFile() {
-    this.date = Time.template('yyyy-MM-dd')
-    this.writer = new FileWriter(`${this.root}/${this.date}.log`)
+  async createFile(date: string, index: number) {
+    this.writer = new FileWriter(date, `${this.root}/${date}-${index}.log`)
 
     const { maxAge } = this.config
     if (!maxAge) return
 
-    const current = Time.getDateNumber(new Date(), 0)
-    this.files = this.files.filter((date) => {
-      if (date >= current - maxAge) return true
-      rm(`${this.root}/${Time.template('yyyy-MM-dd', Time.fromDateNumber(date, 0))}.log`)
-    })
+    const now = Date.now()
+    for (const date in this.files) {
+      if (now - +new Date(date) < maxAge * Time.day) continue
+      for (let index = 1; index <= this.files[date]; ++index) {
+        await rm(`${this.root}/${date}-${index}.log`)
+      }
+    }
   }
 
   prepareLogger() {
-    if (this.ctx.prologue) {
-      for (const line of this.ctx.prologue) {
-        this.printText(line)
+    if (this.ctx.loader.prolog) {
+      for (const record of this.ctx.loader.prolog) {
+        this.record(record)
       }
-      this.ctx.root.prologue = null
+      this.ctx.root.loader.prolog = null
     }
 
     const target: Logger.Target = {
       colors: 3,
-      showTime: 'yyyy-MM-dd hh:mm:ss',
-      print: this.printText.bind(this),
+      record: this.record.bind(this),
     }
 
     Logger.targets.push(target)
@@ -86,13 +87,18 @@ class LogProvider extends DataService<string[]> {
     })
   }
 
-  printText(text: string) {
-    if (!text.startsWith(this.date)) {
+  record(record: Logger.Record) {
+    const date = new Date(record.timestamp).toISOString().slice(0, 10)
+    if (this.writer.date !== date) {
       this.writer.close()
-      this.createFile()
+      this.createFile(date, this.files[date] = 1)
+    }
+    this.writer.write(record)
+    this.patch([record])
+    if (this.writer.size >= this.config.maxSize) {
+      this.writer.close()
+      this.createFile(date, ++this.files[date])
     }
-    this.writer.write(text)
-    this.patch([text])
   }
 
   async get() {
@@ -104,41 +110,17 @@ namespace LogProvider {
   export interface Config {
     root?: string
     maxAge?: number
+    maxSize?: number
   }
 
   export const Config: Schema<Config> = Schema.object({
-    root: Schema.string().default('logs').description('存放输出日志的本地目录。'),
+    root: Schema.path({
+      filters: ['directory'],
+      allowCreate: true,
+    }).default('data/logs').description('存放输出日志的本地目录。'),
     maxAge: Schema.natural().default(30).description('日志文件保存的最大天数。'),
+    maxSize: Schema.natural().default(1024 * 100).description('单个日志文件的最大大小。'),
   })
 }
 
 export default LogProvider
-
-class FileWriter {
-  private task: Promise<FileHandle>
-  private content: string[] = []
-
-  constructor(path: string) {
-    this.task = open(path, 'a+').then(async (handle) => {
-      const text = await handle.readFile('utf-8')
-      if (text) this.content = text.split(/\n(?=\S)/g)
-      return handle
-    })
-  }
-
-  async read() {
-    await this.task
-    return this.content
-  }
-
-  async write(text: string) {
-    const handle = await this.task
-    await handle.write(text + '\n')
-    this.content.push(text)
-  }
-
-  async close() {
-    const handle = await this.task
-    await handle.close()
-  }
-}