From 5af67e82d5dc36e1d7b11cc11cee09520fec0fc7 Mon Sep 17 00:00:00 2001 From: Blake Byrnes Date: Thu, 6 Jul 2023 17:59:14 -0400 Subject: [PATCH] fix(core): add db retention to session registry --- core/dbs/DefaultSessionRegistry.ts | 65 +++++++--- core/dbs/SessionDb.ts | 181 +++++++++++++++------------- core/interfaces/ISessionRegistry.ts | 3 +- core/lib/Session.ts | 8 +- timetravel/lib/DomStateGenerator.ts | 23 +--- 5 files changed, 152 insertions(+), 128 deletions(-) diff --git a/core/dbs/DefaultSessionRegistry.ts b/core/dbs/DefaultSessionRegistry.ts index 1227e2fd8..1e11cf13f 100644 --- a/core/dbs/DefaultSessionRegistry.ts +++ b/core/dbs/DefaultSessionRegistry.ts @@ -6,7 +6,9 @@ import ISessionRegistry from '../interfaces/ISessionRegistry'; import SessionDb from './SessionDb'; export default class DefaultSessionRegistry implements ISessionRegistry { - private byId: { [sessionId: string]: SessionDb } = {}; + private byId: { + [sessionId: string]: { db: SessionDb; deleteRequested?: boolean; connections: number }; + } = {}; constructor(public defaultDir: string) { if (!Fs.existsSync(this.defaultDir)) Fs.mkdirSync(this.defaultDir, { recursive: true }); @@ -16,7 +18,7 @@ export default class DefaultSessionRegistry implements ISessionRegistry { public create(sessionId: string, customPath?: string): SessionDb { const dbPath = this.resolvePath(sessionId, customPath); const db = new SessionDb(sessionId, dbPath); - this.byId[sessionId] = db; + this.byId[sessionId] = { db, connections: 1 }; return db; } @@ -35,29 +37,60 @@ export default class DefaultSessionRegistry implements ISessionRegistry { // eslint-disable-next-line @typescript-eslint/require-await public async get(sessionId: string, customPath?: string): Promise { if (sessionId.endsWith('.db')) sessionId = sessionId.slice(0, -3); - if (!this.byId[sessionId]?.isOpen || this.byId[sessionId]?.isClosing) { + const entry = this.byId[sessionId]; + if (!entry?.db?.isOpen || entry?.connections === 0) { const dbPath = this.resolvePath(sessionId, customPath); - this.byId[sessionId] = new SessionDb(sessionId, dbPath, { - readonly: true, - fileMustExist: true, - }); + this.byId[sessionId] = { + db: new SessionDb(sessionId, dbPath, { + readonly: true, + fileMustExist: true, + }), + connections: 1, + }; } - return this.byId[sessionId]; + return this.byId[sessionId]?.db; } - public async onClosed(sessionId: string, isDeleteRequested: boolean): Promise { + public async retain(sessionId: string, customPath?: string): Promise { + if (sessionId.endsWith('.db')) sessionId = sessionId.slice(0, -3); const entry = this.byId[sessionId]; - delete this.byId[sessionId]; - if (entry && isDeleteRequested) { - try { - await Fs.promises.rm(entry.path); - } catch {} + if (!entry?.db?.isOpen) { + return this.get(sessionId, customPath); + } + + if (entry) { + entry.connections += 1; + return entry.db; + } + } + + public async close(sessionId: string, isDeleteRequested: boolean): Promise { + const entry = this.byId[sessionId]; + if (!entry) return; + entry.connections -= 1; + entry.deleteRequested ||= isDeleteRequested; + + if (entry.connections < 1) { + delete this.byId[sessionId]; + entry.db.close(); + if (entry.deleteRequested) { + try { + await Fs.promises.rm(entry.db.path); + } catch {} + } + } else if (!entry.db?.readonly) { + entry.db.recycle(); } } - public shutdown(): Promise { + public async shutdown(): Promise { for (const [key, value] of Object.entries(this.byId)) { - value.close(); + value.db.close(); + if (value.deleteRequested) { + try { + await Fs.promises.rm(value.db.path); + } catch {} + } delete this.byId[key]; } return Promise.resolve(); diff --git a/core/dbs/SessionDb.ts b/core/dbs/SessionDb.ts index 73073984c..3c2a63f70 100644 --- a/core/dbs/SessionDb.ts +++ b/core/dbs/SessionDb.ts @@ -48,34 +48,33 @@ export default class SessionDb { public readonly path: string; - public readonly commands: CommandsTable; - public readonly frames: FramesTable; - public readonly frameNavigations: FrameNavigationsTable; - public readonly sockets: SocketsTable; - public readonly resources: ResourcesTable; - public readonly resourceStates: ResourceStatesTable; - public readonly websocketMessages: WebsocketMessagesTable; - public readonly domChanges: DomChangesTable; - public readonly detachedElements: DetachedElementsTable; - public readonly detachedResources: DetachedResourcesTable; - public readonly snippets: SnippetsTable; - public readonly interactions: InteractionStepsTable; - public readonly flowHandlers: FlowHandlersTable; - public readonly flowCommands: FlowCommandsTable; - public readonly pageLogs: PageLogsTable; - public readonly sessionLogs: SessionLogsTable; - public readonly session: SessionTable; - public readonly mouseEvents: MouseEventsTable; - public readonly focusEvents: FocusEventsTable; - public readonly scrollEvents: ScrollEventsTable; - public readonly storageChanges: StorageChangesTable; - public readonly screenshots: ScreenshotsTable; - public readonly devtoolsMessages: DevtoolsMessagesTable; - public readonly awaitedEvents: AwaitedEventsTable; - public readonly tabs: TabsTable; - public readonly output: OutputTable; + public commands: CommandsTable; + public frames: FramesTable; + public frameNavigations: FrameNavigationsTable; + public sockets: SocketsTable; + public resources: ResourcesTable; + public resourceStates: ResourceStatesTable; + public websocketMessages: WebsocketMessagesTable; + public domChanges: DomChangesTable; + public detachedElements: DetachedElementsTable; + public detachedResources: DetachedResourcesTable; + public snippets: SnippetsTable; + public interactions: InteractionStepsTable; + public flowHandlers: FlowHandlersTable; + public flowCommands: FlowCommandsTable; + public pageLogs: PageLogsTable; + public sessionLogs: SessionLogsTable; + public session: SessionTable; + public mouseEvents: MouseEventsTable; + public focusEvents: FocusEventsTable; + public scrollEvents: ScrollEventsTable; + public storageChanges: StorageChangesTable; + public screenshots: ScreenshotsTable; + public devtoolsMessages: DevtoolsMessagesTable; + public awaitedEvents: AwaitedEventsTable; + public tabs: TabsTable; + public output: OutputTable; public readonly sessionId: string; - public isClosing = false; public keepAlive = false; @@ -98,61 +97,7 @@ export default class SessionDb { this.saveInterval = setInterval(this.flush.bind(this), 5e3).unref(); } - this.commands = new CommandsTable(this.db); - this.tabs = new TabsTable(this.db); - this.frames = new FramesTable(this.db); - this.frameNavigations = new FrameNavigationsTable(this.db); - this.sockets = new SocketsTable(this.db); - this.resources = new ResourcesTable(this.db); - this.resourceStates = new ResourceStatesTable(this.db); - this.websocketMessages = new WebsocketMessagesTable(this.db); - this.domChanges = new DomChangesTable(this.db); - this.detachedElements = new DetachedElementsTable(this.db); - this.detachedResources = new DetachedResourcesTable(this.db); - this.snippets = new SnippetsTable(this.db); - this.flowHandlers = new FlowHandlersTable(this.db); - this.flowCommands = new FlowCommandsTable(this.db); - this.pageLogs = new PageLogsTable(this.db); - this.session = new SessionTable(this.db); - this.interactions = new InteractionStepsTable(this.db); - this.mouseEvents = new MouseEventsTable(this.db); - this.focusEvents = new FocusEventsTable(this.db); - this.scrollEvents = new ScrollEventsTable(this.db); - this.sessionLogs = new SessionLogsTable(this.db); - this.screenshots = new ScreenshotsTable(this.db); - this.storageChanges = new StorageChangesTable(this.db); - this.devtoolsMessages = new DevtoolsMessagesTable(this.db); - this.awaitedEvents = new AwaitedEventsTable(this.db); - this.output = new OutputTable(this.db); - - this.tables.push( - this.commands, - this.tabs, - this.frames, - this.frameNavigations, - this.sockets, - this.resources, - this.resourceStates, - this.websocketMessages, - this.domChanges, - this.detachedElements, - this.detachedResources, - this.snippets, - this.flowHandlers, - this.flowCommands, - this.pageLogs, - this.session, - this.interactions, - this.mouseEvents, - this.focusEvents, - this.scrollEvents, - this.sessionLogs, - this.devtoolsMessages, - this.screenshots, - this.storageChanges, - this.awaitedEvents, - this.output, - ); + this.attach(); if (!readonly) { this.batchInsert = this.db.transaction(() => { @@ -201,18 +146,26 @@ export default class SessionDb { this.flush(); } + if (env.enableSqliteWal && !this.db.readonly) { + this.db.pragma('journal_mode = DELETE'); + } + if (this.keepAlive) { this.db.readonly = true; return; } - if (env.enableSqliteWal && !this.db.readonly) { - this.db.pragma('journal_mode = DELETE'); - } this.db.close(); this.db = null; } + public recycle(): void { + this.close(); + + this.db = new Database(this.path, { readonly: true }); + this.attach(); + } + public flush(): void { if (this.batchInsert) { try { @@ -228,4 +181,62 @@ export default class SessionDb { } } } + + private attach(): void { + this.commands = new CommandsTable(this.db); + this.tabs = new TabsTable(this.db); + this.frames = new FramesTable(this.db); + this.frameNavigations = new FrameNavigationsTable(this.db); + this.sockets = new SocketsTable(this.db); + this.resources = new ResourcesTable(this.db); + this.resourceStates = new ResourceStatesTable(this.db); + this.websocketMessages = new WebsocketMessagesTable(this.db); + this.domChanges = new DomChangesTable(this.db); + this.detachedElements = new DetachedElementsTable(this.db); + this.detachedResources = new DetachedResourcesTable(this.db); + this.snippets = new SnippetsTable(this.db); + this.flowHandlers = new FlowHandlersTable(this.db); + this.flowCommands = new FlowCommandsTable(this.db); + this.pageLogs = new PageLogsTable(this.db); + this.session = new SessionTable(this.db); + this.interactions = new InteractionStepsTable(this.db); + this.mouseEvents = new MouseEventsTable(this.db); + this.focusEvents = new FocusEventsTable(this.db); + this.scrollEvents = new ScrollEventsTable(this.db); + this.sessionLogs = new SessionLogsTable(this.db); + this.screenshots = new ScreenshotsTable(this.db); + this.storageChanges = new StorageChangesTable(this.db); + this.devtoolsMessages = new DevtoolsMessagesTable(this.db); + this.awaitedEvents = new AwaitedEventsTable(this.db); + this.output = new OutputTable(this.db); + + this.tables.push( + this.commands, + this.tabs, + this.frames, + this.frameNavigations, + this.sockets, + this.resources, + this.resourceStates, + this.websocketMessages, + this.domChanges, + this.detachedElements, + this.detachedResources, + this.snippets, + this.flowHandlers, + this.flowCommands, + this.pageLogs, + this.session, + this.interactions, + this.mouseEvents, + this.focusEvents, + this.scrollEvents, + this.sessionLogs, + this.devtoolsMessages, + this.screenshots, + this.storageChanges, + this.awaitedEvents, + this.output, + ); + } } diff --git a/core/interfaces/ISessionRegistry.ts b/core/interfaces/ISessionRegistry.ts index a58f4d16e..4c183d76f 100644 --- a/core/interfaces/ISessionRegistry.ts +++ b/core/interfaces/ISessionRegistry.ts @@ -3,8 +3,9 @@ import SessionDb from '../dbs/SessionDb'; export default interface ISessionRegistry { defaultDir: string; ids(): Promise; + retain(sessionId: string, customPath?: string): Promise; get(sessionId: string, customPath?: string): Promise; create(sessionId: string, customPath?: string): SessionDb; - onClosed(sessionId: string, isDeleteRequested: boolean): Promise; + close(sessionId: string, isDeleteRequested: boolean): Promise; shutdown(): Promise } diff --git a/core/lib/Session.ts b/core/lib/Session.ts index a3efeab4a..c30d840c5 100644 --- a/core/lib/Session.ts +++ b/core/lib/Session.ts @@ -263,7 +263,6 @@ export default class Session const customPath = this.getCustomSessionPath(fromSessionId); db = await this.sessionRegistry.get(fromSessionId, customPath); } - db.flush(); return DetachedAssets.getElements(db, name); } @@ -450,7 +449,6 @@ export default class Session }); const closedEvent = { waitForPromise: null }; - this.db.isClosing = true; try { this.emit('closed', closedEvent); await closedEvent.waitForPromise; @@ -477,11 +475,7 @@ export default class Session this.removeAllListeners(); try { - this.db.close(); - } catch {} - - try { - await sessionRegistry.onClosed(this.id, this.options.sessionPersistence === false); + await sessionRegistry.close(this.id, this.options.sessionPersistence === false); } catch (e) { /* no-op */ } diff --git a/timetravel/lib/DomStateGenerator.ts b/timetravel/lib/DomStateGenerator.ts index 16fb4e5c9..b1b331154 100644 --- a/timetravel/lib/DomStateGenerator.ts +++ b/timetravel/lib/DomStateGenerator.ts @@ -51,16 +51,13 @@ export default class DomStateGenerator { timelineRange?: [start: number, end: number], ): void { const sessionId = sessionDb.sessionId; - const getSessionDb = this.getSessionDb.bind(this, sessionId); + const db = this.sessionRegistry.retain(sessionId).catch(() => null); this.sessionsById.set(sessionId, { tabId, sessionId, needsProcessing: true, mainFrameIds: sessionDb.frames.mainFrameIds(), - // could get closed, so need to use getter - get db() { - return getSessionDb(); - }, + db, dbLocation: Path.dirname(sessionDb.path), loadingRange: [...loadingRange], timelineRange: timelineRange ? [...timelineRange] : undefined, @@ -97,19 +94,11 @@ export default class DomStateGenerator { } for (const session of savedState.sessions) { const sessionId = session.sessionId; - let db: SessionDb; - try { - db = await this.sessionRegistry.get(sessionId).catch(() => null); - } catch (err) { - // couldn't load - } + const db = await this.sessionRegistry.retain(sessionId).catch(() => null as SessionDb); - const getSessionDb = this.getSessionDb.bind(this, sessionId); this.sessionsById.set(sessionId, { ...session, - get db(): Promise { - return getSessionDb(); - }, + db: Promise.resolve(db), needsProcessing: !!db, mainFrameIds: db?.frames.mainFrameIds(session.tabId), }); @@ -375,10 +364,6 @@ export default class DomStateGenerator { } } - private getSessionDb(sessionId: string): Promise { - return this.sessionRegistry.get(sessionId).catch(() => null); - } - private processStorageChanges( changes: IStorageChangesEntry[], sessionId: string,