From 69db46e40cb2a8980337d9714419cc4d045b1dbc Mon Sep 17 00:00:00 2001 From: Blake Byrnes Date: Mon, 17 Jan 2022 17:55:20 -0500 Subject: [PATCH] feat(core): collect and recreate fragments --- client/lib/CoreFrameEnvironment.ts | 4 ++ client/lib/CoreSession.ts | 50 ++++++++------ client/lib/DomExtender.ts | 19 ++++-- client/lib/Fragment.ts | 37 +++++++++++ client/lib/FrozenFrameEnvironment.ts | 2 +- client/lib/Hero.ts | 63 ++++++++++-------- core/dbs/SessionDb.ts | 4 ++ core/lib/DetachedTabState.ts | 14 ++-- core/lib/FrameEnvironment.ts | 34 ++++++++-- core/lib/JsPath.ts | 15 ++++- core/lib/Session.ts | 73 +++++++++++++++------ core/lib/Tab.ts | 1 - core/models/DomChangesTable.ts | 11 ++++ core/models/FragmentsTable.ts | 48 ++++++++++++++ core/models/FrameNavigationsTable.ts | 10 +++ fullstack/test/fragments.test.ts | 98 ++++++++++++++++++++++++++++ 16 files changed, 397 insertions(+), 86 deletions(-) create mode 100644 client/lib/Fragment.ts create mode 100644 core/models/FragmentsTable.ts create mode 100644 fullstack/test/fragments.test.ts diff --git a/client/lib/CoreFrameEnvironment.ts b/client/lib/CoreFrameEnvironment.ts index cc04f8779..b7979117c 100644 --- a/client/lib/CoreFrameEnvironment.ts +++ b/client/lib/CoreFrameEnvironment.ts @@ -82,6 +82,10 @@ export default class CoreFrameEnvironment { return await this.commandQueue.run('FrameEnvironment.createRequest', input, init); } + public async createFragment(name: string, jsPath: IJsPath): Promise { + return await this.commandQueue.run('FrameEnvironment.createFragment', name, jsPath); + } + public async getUrl(): Promise { return await this.commandQueue.run('FrameEnvironment.getUrl'); } diff --git a/client/lib/CoreSession.ts b/client/lib/CoreSession.ts index 8db4e021f..66845bd0e 100644 --- a/client/lib/CoreSession.ts +++ b/client/lib/CoreSession.ts @@ -16,6 +16,7 @@ import IJsPathEventTarget from '../interfaces/IJsPathEventTarget'; import ConnectionToCore from '../connections/ConnectionToCore'; import ICommandCounter from '../interfaces/ICommandCounter'; import ISessionCreateOptions from '@ulixee/hero-interfaces/ISessionCreateOptions'; +import INodePointer from 'awaited-dom/base/INodePointer'; export default class CoreSession implements IJsPathEventTarget { public tabsById = new Map(); @@ -141,27 +142,36 @@ export default class CoreSession implements IJsPathEventTarget { }; } - // @experimental - public async loadFrozenTab( - sessionId: string, - name: string, - atCommandId: number, - tabId = 1, - ): Promise<{ coreTab: CoreTab; prefetchedJsPaths: IJsPathResult[] }> { - const { detachedTab, prefetchedJsPaths } = await this.commandQueue.runOutOfBand<{ - detachedTab: ISessionMeta; + public async loadFragments(sessionId: string): Promise< + { + name: string; + nodePointer: INodePointer; + coreTab: CoreTab; prefetchedJsPaths: IJsPathResult[]; - }>('Session.loadFrozenTab', sessionId, name, atCommandId, tabId); - const coreTab = new CoreTab( - { ...detachedTab, sessionName: this.sessionName }, - this.connectionToCore, - this, - ); - this.frozenTabsById.set(detachedTab.tabId, coreTab); - return { - coreTab, - prefetchedJsPaths, - }; + }[] + > { + const fragments = await this.commandQueue.run< + { + name: string; + nodePointer: INodePointer; + detachedTab: ISessionMeta; + prefetchedJsPaths: IJsPathResult[]; + }[] + >('Session.loadAllFragments', sessionId); + return fragments.map(fragment => { + const coreTab = new CoreTab( + { ...fragment.detachedTab, sessionName: this.sessionName }, + this.connectionToCore, + this, + ); + this.frozenTabsById.set(fragment.detachedTab.tabId, coreTab); + return { + name: fragment.name, + nodePointer: fragment.nodePointer, + coreTab, + prefetchedJsPaths: fragment.prefetchedJsPaths, + }; + }); } public async close(force = false): Promise { diff --git a/client/lib/DomExtender.ts b/client/lib/DomExtender.ts index 89a648734..e0f5a7bc2 100644 --- a/client/lib/DomExtender.ts +++ b/client/lib/DomExtender.ts @@ -30,15 +30,14 @@ interface IBaseExtendNode { $isClickable: Promise; $clearValue(): Promise; $click(verification?: IElementInteractVerification): Promise; + $extractLater(name: string): Promise; $type(...typeInteractions: ITypeInteraction[]): Promise; $waitForHidden(options?: { timeoutMs?: number }): Promise; $waitForVisible(options?: { timeoutMs?: number }): Promise; } interface IBaseExtendNodeList { - $map( - iteratorFn: (node: ISuperNode, index: number) => Promise - ): Promise; + $map(iteratorFn: (node: ISuperNode, index: number) => Promise): Promise; $reduce( iteratorFn: (initial: T, node: ISuperNode) => Promise, initial: T, @@ -66,6 +65,11 @@ const NodeExtensionFns: Partial = { const coreFrame = await getCoreFrame(this); await Interactor.run(coreFrame, [{ click: { element: this, verification } }]); }, + async $extractLater(name: string): Promise { + const { awaitedPath, awaitedOptions } = awaitedPathState.getState(this); + const coreFrame = await awaitedOptions.coreFrame; + await coreFrame.createFragment(name, awaitedPath.toJSON()); + }, async $type(...typeInteractions: ITypeInteraction[]): Promise { const coreFrame = await getCoreFrame(this); await this.$click(); @@ -142,14 +146,17 @@ const NodeListExtensionFns: IBaseExtendNodeList = { } return newArray; }, - async $reduce(iteratorFn: (initial: T, node: ISuperNode) => Promise, initial: T): Promise { + async $reduce( + iteratorFn: (initial: T, node: ISuperNode) => Promise, + initial: T, + ): Promise { const nodes = await this; for (const node of nodes) { initial = await iteratorFn(initial, node); } return initial; - } -} + }, +}; for (const Item of [SuperElement, SuperNode, SuperHTMLElement, Element, Node, HTMLElement]) { for (const [key, value] of Object.entries(NodeExtensionFns)) { diff --git a/client/lib/Fragment.ts b/client/lib/Fragment.ts new file mode 100644 index 000000000..b9da2a16d --- /dev/null +++ b/client/lib/Fragment.ts @@ -0,0 +1,37 @@ +import AwaitedPath from 'awaited-dom/base/AwaitedPath'; +import { ISuperElement } from 'awaited-dom/base/interfaces/super'; +import FrozenTab from './FrozenTab'; +import { createInstanceWithNodePointer } from './SetupAwaitedHandler'; +import { getState as getFrozenFrameState } from './FrozenFrameEnvironment'; +import INodePointer from 'awaited-dom/base/INodePointer'; +import StateMachine from 'awaited-dom/base/StateMachine'; +import IAwaitedOptions from '../interfaces/IAwaitedOptions'; + +const awaitedPathState = StateMachine< + any, + { awaitedPath: AwaitedPath; awaitedOptions: IAwaitedOptions; nodePointer?: INodePointer } +>(); + +export default class Fragment { + public element: ISuperElement; + public name: string; + + readonly #nodePointer: INodePointer; + readonly #frozenTab: FrozenTab; + + constructor(frozenTab: FrozenTab, name: string, nodePointer: INodePointer) { + this.name = name; + this.#frozenTab = frozenTab; + this.#nodePointer = nodePointer; + this.element = createInstanceWithNodePointer( + awaitedPathState, + new AwaitedPath(null), + getFrozenFrameState(frozenTab.mainFrameEnvironment), + nodePointer, + ); + } + + public close(): Promise { + return this.#frozenTab.close(); + } +} diff --git a/client/lib/FrozenFrameEnvironment.ts b/client/lib/FrozenFrameEnvironment.ts index f729c1e8d..17cedd4f9 100644 --- a/client/lib/FrozenFrameEnvironment.ts +++ b/client/lib/FrozenFrameEnvironment.ts @@ -29,7 +29,7 @@ import CoreFrameEnvironment from './CoreFrameEnvironment'; import FrozenTab from './FrozenTab'; import * as AwaitedHandler from './SetupAwaitedHandler'; -const { getState, setState } = StateMachine(); +export const { getState, setState } = StateMachine(); const awaitedPathState = StateMachine< any, { awaitedPath: AwaitedPath; awaitedOptions: IAwaitedOptions } diff --git a/client/lib/Hero.ts b/client/lib/Hero.ts index ce00b216d..58b5de2db 100644 --- a/client/lib/Hero.ts +++ b/client/lib/Hero.ts @@ -65,6 +65,7 @@ import ConnectionManager from './ConnectionManager'; import './DomExtender'; import IPageStateDefinitions from '../interfaces/IPageStateDefinitions'; import IMagicSelectorOptions from '@ulixee/hero-interfaces/IMagicSelectorOptions'; +import Fragment from './Fragment'; export const DefaultOptions = { defaultBlockedResourceTypes: [BlockedResourceType.None], @@ -79,8 +80,6 @@ export type IStateOptions = Omit(); + readonly #options: IStateOptions; + #isClosing = false; + constructor(options: IHeroCreateOptions = {}) { - const connectionManagerIsReady = createPromise(); super(async () => { - await connectionManagerIsReady.promise; + await this.#connectManagerIsReady.promise; return { target: getState(this).connection.getConnectedCoreSessionOrReject(), }; @@ -133,7 +136,7 @@ export default class Hero extends AwaitedEventTarget<{ const sessionName = scriptInstance.generateSessionName(options.name); delete options.name; - options = { + this.#options = { ...options, mode: options.mode ?? scriptInstance.mode, sessionName, @@ -142,16 +145,14 @@ export default class Hero extends AwaitedEventTarget<{ corePluginPaths: [], } as IStateOptions; - const connection = new ConnectionManager(this, options); + const connection = new ConnectionManager(this, this.#options); setState(this, { connection, - isClosing: false, - options, clientPlugins: [], }); - connectionManagerIsReady.resolve(); + this.#connectManagerIsReady.resolve(); } public get activeTab(): Tab { @@ -192,7 +193,7 @@ export default class Hero extends AwaitedEventTarget<{ } public get sessionName(): Promise { - return Promise.resolve(getState(this).options.sessionName); + return Promise.resolve(this.#options.sessionName); } public get meta(): Promise { @@ -227,9 +228,9 @@ export default class Hero extends AwaitedEventTarget<{ // METHODS public async close(): Promise { - const { isClosing, connection } = getState(this); - if (isClosing) return; - setState(this, { isClosing: true }); + const { connection } = getState(this); + if (this.#isClosing) return; + this.#isClosing = true; try { return await connection.close(); @@ -243,19 +244,25 @@ export default class Hero extends AwaitedEventTarget<{ await tab.close(); } - // @experimental - public loadFrozenTabs( - sessionId: string, - tabNameToCommandId: { [name: string]: number }, - ): { [name: string]: FrozenTab } { - const coreSession = getState(this).connection.getConnectedCoreSessionOrReject(); + public getFragment(name: string): T { + return this.#fragmentsByName.get(name).element as T; + } - const result: { [name: string]: FrozenTab } = {}; - for (const [name, commandId] of Object.entries(tabNameToCommandId)) { - const coreFrozenTab = coreSession.then(x => x.loadFrozenTab(sessionId, name, commandId)); - result[name] = new FrozenTab(this, coreFrozenTab); - } - return result; + public async importFragments(sessionId: string): Promise { + const coreSession = await getState(this).connection.getConnectedCoreSessionOrReject(); + const fragments = await coreSession.loadFragments(sessionId); + return fragments.map(x => { + const frozenTab = new FrozenTab( + this, + Promise.resolve({ + coreTab: x.coreTab, + prefetchedJsPaths: x.prefetchedJsPaths, + }), + ); + const fragment = new Fragment(frozenTab, x.name, x.nodePointer); + this.#fragmentsByName.set(fragment.name, fragment); + return fragment; + }); } public detach(tab: Tab, key?: string): FrozenTab { @@ -340,7 +347,7 @@ export default class Hero extends AwaitedEventTarget<{ // PLUGINS public use(PluginObject: string | IClientPluginClass | { [name: string]: IPluginClass }): void { - const { clientPlugins, options, connection } = getState(this); + const { clientPlugins, connection } = getState(this); const ClientPluginsById: { [id: string]: IClientPluginClass } = {}; if (connection.hasConnected) { @@ -354,7 +361,7 @@ export default class Hero extends AwaitedEventTarget<{ const CorePlugins = filterPlugins(Plugins, PluginTypes.CorePlugin); const ClientPlugins = filterPlugins(Plugins, PluginTypes.ClientPlugin); if (CorePlugins.length) { - options.corePluginPaths.push(PluginObject); + this.#options.corePluginPaths.push(PluginObject); } ClientPlugins.forEach(ClientPlugin => (ClientPluginsById[ClientPlugin.id] = ClientPlugin)); } else { @@ -372,7 +379,7 @@ export default class Hero extends AwaitedEventTarget<{ clientPlugin.onHero(this, connection.sendToActiveTab); } - options.dependencyMap[ClientPlugin.id] = ClientPlugin.coreDependencyIds || []; + this.#options.dependencyMap[ClientPlugin.id] = ClientPlugin.coreDependencyIds || []; }); } diff --git a/core/dbs/SessionDb.ts b/core/dbs/SessionDb.ts index 62f1e7200..8dad509de 100644 --- a/core/dbs/SessionDb.ts +++ b/core/dbs/SessionDb.ts @@ -24,6 +24,7 @@ import SocketsTable from '../models/SocketsTable'; import Core from '../index'; import StorageChangesTable from '../models/StorageChangesTable'; import AwaitedEventsTable from '../models/AwaitedEventsTable'; +import FragmentsTable from '../models/FragmentsTable'; const { log } = Log(module); @@ -48,6 +49,7 @@ export default class SessionDb { public readonly resourceStates: ResourceStatesTable; public readonly websocketMessages: WebsocketMessagesTable; public readonly domChanges: DomChangesTable; + public readonly fragments: FragmentsTable; public readonly pageLogs: PageLogsTable; public readonly sessionLogs: SessionLogsTable; public readonly session: SessionTable; @@ -87,6 +89,7 @@ export default class SessionDb { this.resourceStates = new ResourceStatesTable(this.db); this.websocketMessages = new WebsocketMessagesTable(this.db); this.domChanges = new DomChangesTable(this.db); + this.fragments = new FragmentsTable(this.db); this.pageLogs = new PageLogsTable(this.db); this.session = new SessionTable(this.db); this.mouseEvents = new MouseEventsTable(this.db); @@ -108,6 +111,7 @@ export default class SessionDb { this.resourceStates, this.websocketMessages, this.domChanges, + this.fragments, this.pageLogs, this.session, this.mouseEvents, diff --git a/core/lib/DetachedTabState.ts b/core/lib/DetachedTabState.ts index 8f3ed6c95..99415752e 100644 --- a/core/lib/DetachedTabState.ts +++ b/core/lib/DetachedTabState.ts @@ -14,6 +14,7 @@ import { IJsPathHistory } from './JsPath'; import SessionDb from '../dbs/SessionDb'; import IJsPathResult from '@ulixee/hero-interfaces/IJsPathResult'; import { IPuppetPage } from '@ulixee/hero-interfaces/IPuppetPage'; +import IPuppetContext from '@ulixee/hero-interfaces/IPuppetContext'; export default class DetachedTabState { public get url(): string { @@ -52,7 +53,7 @@ export default class DetachedTabState { readonly detachedAtCommandId: number, private readonly initialPageNavigation: INavigation, domChangeRecords: IDomChangeRecord[], - readonly callsite: string, + readonly callsitePath: string, readonly key?: string, ) { this.mirrorNetwork = DetachedTabState.createMirrorNetwork(sourceTabId, sessionDb); @@ -74,10 +75,11 @@ export default class DetachedTabState { public async openInNewTab( viewport: IViewport, + context?: IPuppetContext, label?: string, ): Promise<{ detachedTab: Tab; prefetchedJsPaths: IJsPathResult[] }> { await this.mirrorPage.open( - this.activeSession.browserContext, + context ?? this.activeSession.browserContext, this.sessionDb.sessionId, viewport, this.onNewPuppetPage.bind(this), @@ -117,7 +119,11 @@ export default class DetachedTabState { public getJsPathHistory(): IJsPathHistory[] { const { scriptInstanceMeta } = this.activeSession.options; - return SessionsDb.find().findDetachedJsPathCalls(scriptInstanceMeta, this.callsite, this.key); + return SessionsDb.find().findDetachedJsPathCalls( + scriptInstanceMeta, + this.callsitePath, + this.key, + ); } public saveHistory(history: IJsPathHistory[]): void { @@ -125,7 +131,7 @@ export default class DetachedTabState { SessionsDb.find().recordDetachedJsPathCalls( scriptInstanceMeta, history, - this.callsite, + this.callsitePath, this.key, ); } diff --git a/core/lib/FrameEnvironment.ts b/core/lib/FrameEnvironment.ts index 9316a5d85..f51094e05 100644 --- a/core/lib/FrameEnvironment.ts +++ b/core/lib/FrameEnvironment.ts @@ -40,6 +40,7 @@ import { PageRecorderResultSet } from '../injected-scripts/pageEventsRecorder'; import { ICommandableTarget } from './CommandRunner'; import { IRemoteEmitFn, IRemoteEventListener } from '../interfaces/IRemoteEventListener'; import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta'; +import { IFragment } from '../models/FragmentsTable'; const { log } = Log(module); @@ -108,6 +109,7 @@ export default class FrameEnvironment private readonly commandRecorder: CommandRecorder; private readonly cleanPaths: string[] = []; private lastDomChangeNavigationId: number; + private lastDomChangeIndex = 0; private isTrackingMouse = false; private readonly installedDomAssertions = new Set(); @@ -143,6 +145,7 @@ export default class FrameEnvironment process.nextTick(() => this.listen()); this.commandRecorder = new CommandRecorder(this, tab.session, tab.id, this.id, [ this.createRequest, + this.createFragment, this.execJsPath, this.fetch, this.getChildFrameEnvironment, @@ -253,6 +256,23 @@ export default class FrameEnvironment return this.toJSON(); } + public async createFragment(name: string, jsPath: IJsPath): Promise { + const { nodePointer } = await this.jsPath.getNodePointer(jsPath); + await this.flushPageEventsRecorder(); + const navigation = this.navigations.lastHttpNavigation; + const fragment: IFragment = { + name, + nodePointerId: nodePointer.id, + nodeType: nodePointer.type, + nodePreview: nodePointer.preview, + frameNavigationId: navigation.id, + commandId: this.session.commands.lastId, + domChangeEventIndex: this.lastDomChangeIndex, + }; + this.session.db.fragments.insert(fragment); + return fragment; + } + public async execJsPath(jsPath: IJsPath): Promise> { // if nothing loaded yet, return immediately if (!this.navigations.top) return null; @@ -566,8 +586,11 @@ b) Use the UserProfile feature to set cookies for 1 or more domains before they' const db = this.session.db; for (const domChange of domChanges) { - lastCommand = commands.getCommandForTimestamp(lastCommand, domChange[2]); - if (domChange[0] === DomActionType.newDocument || domChange[0] === DomActionType.location) { + const [action, nodeData, timestamp, index] = domChange; + lastCommand = commands.getCommandForTimestamp(lastCommand, timestamp); + if (index > this.lastDomChangeIndex) this.lastDomChangeIndex = index; + + if (action === DomActionType.newDocument || action === DomActionType.location) { const url = domChange[1].textContent; navigation = this.navigations.findHistory(x => x.finalUrl === url); @@ -577,11 +600,14 @@ b) Use the UserProfile feature to set cookies for 1 or more domains before they' ) { this.lastDomChangeNavigationId = navigation.id; } + if (index < this.lastDomChangeIndex) { + this.lastDomChangeIndex = index; + } } // if this is a doctype, set into the navigation - if (navigation && domChange[0] === DomActionType.added && domChange[1].nodeType === 10) { - navigation.doctype = domChange[1].textContent; + if (navigation && action === DomActionType.added && nodeData.nodeType === 10) { + navigation.doctype = nodeData.textContent; db.frameNavigations.insert(navigation); } diff --git a/core/lib/JsPath.ts b/core/lib/JsPath.ts index 0696433bd..4e8667253 100644 --- a/core/lib/JsPath.ts +++ b/core/lib/JsPath.ts @@ -77,7 +77,13 @@ export class JsPath { } } - // add a node pointer call onto the end if needed + return this.getNodePointer(jsPath, containerOffset); + } + + public getNodePointer( + jsPath: IJsPath, + containerOffset: IPoint = { x: 0, y: 0 }, + ): Promise> { const fnCall = this.getJsPathMethod(jsPath); if (fnCall !== getClientRectFnName && fnCall !== getNodePointerFnName) { jsPath = [...jsPath, [getNodePointerFnName]]; @@ -163,6 +169,13 @@ export class JsPath { return results; } + public setFragmentNode(nodePointer: INodePointer): void { + this.nodeIdToHistoryLocation.set(nodePointer.id, { sourceIndex: 0, isFromIterable: false }); + this.execHistory.push({ + jsPath: [nodePointer.id, [getNodePointerFnName]], + }); + } + private async runJsPath( fnName: string, jsPath: IJsPath, diff --git a/core/lib/Session.ts b/core/lib/Session.ts index 7276b76d6..f56a34b58 100644 --- a/core/lib/Session.ts +++ b/core/lib/Session.ts @@ -38,6 +38,7 @@ import WebsocketMessages from './WebsocketMessages'; import SessionsDb from '../dbs/SessionsDb'; import { IRemoteEmitFn, IRemoteEventListener } from '../interfaces/IRemoteEventListener'; import DetachedTabState from './DetachedTabState'; +import INodePointer from 'awaited-dom/base/INodePointer'; const { log } = Log(module); @@ -196,6 +197,7 @@ export default class Session this.configure, this.detachTab, this.loadFrozenTab, + this.loadAllFragments, this.close, this.flush, this.exportUserProfile, @@ -267,6 +269,7 @@ export default class Session const result = await detachedState.openInNewTab( this.viewport, + this.browserContext, `Frozen Tab at Command ${detachedState.detachedAtCommandId}`, ); @@ -276,11 +279,46 @@ export default class Session return result; } + public async loadAllFragments(fromSessionId: string): Promise< + { + name: string; + nodePointer: INodePointer; + detachedTab: Tab; + prefetchedJsPaths: IJsPathResult[]; + }[] + > { + await this.db.flush(); + const fragments = this.db.fragments.all(); + return await Promise.all( + fragments.map(async fragment => { + const { name, frameNavigationId, domChangeEventIndex } = fragment; + const { detachedTab, prefetchedJsPaths } = await this.loadFrozenTab( + fromSessionId, + name, + frameNavigationId, + domChangeEventIndex, + ); + const nodePointer: INodePointer = { + id: fragment.nodePointerId, + type: fragment.nodeType, + preview: fragment.nodePreview, + }; + detachedTab.mainFrameEnvironment.jsPath.setFragmentNode(nodePointer); + return { + name, + nodePointer, + detachedTab, + prefetchedJsPaths, + }; + }), + ); + } + public async loadFrozenTab( sessionId: string, - label: string, - atCommandId: number, - sourceTabId = 1, + name: string, + frameNavigationId: number, + domChangeEventIndex: number, ): Promise<{ detachedTab: Tab; prefetchedJsPaths: IJsPathResult[]; @@ -290,38 +328,31 @@ export default class Session throw new Error('This session database could not be found'); } const mainFrameIds = sessionDb.frames.mainFrameIds(); - const navigations = sessionDb.frameNavigations.getAllNavigations(); - let lastLoadedNavigation = navigations[0]; - for (const navigation of navigations) { - if (navigation.startCommandId > atCommandId) break; - if (navigation.tabId !== sourceTabId) continue; - - if (mainFrameIds.has(navigation.frameId) && navigation.statusChanges.has('HttpResponded')) { - lastLoadedNavigation = navigation; - } - } + const navigation = sessionDb.frameNavigations.get(frameNavigationId); - const domChanges = sessionDb.domChanges.getFrameChanges( - lastLoadedNavigation.frameId, - lastLoadedNavigation.startCommandId - 1, + const domChanges = sessionDb.domChanges.getDomChangesForNavigation( + navigation.id, + domChangeEventIndex, ); + const atCommandId = domChanges[domChanges.length - 1].commandId; const detachedState = new DetachedTabState( sessionDb, - sourceTabId, + navigation.tabId, mainFrameIds, this, atCommandId, - lastLoadedNavigation, + navigation, domChanges, - `frozen-at-${atCommandId}`, + name, // name is our lookup key ); const result = await detachedState.openInNewTab( this.viewport, - `Frozen Tab: ${label ?? 'at ' + atCommandId}`, + this.browserContext, + `Frozen Tab: ${name ?? 'at ' + atCommandId}`, ); - this.recordTab(result.detachedTab, sourceTabId, detachedState.detachedAtCommandId); + this.recordTab(result.detachedTab, navigation.tabId, atCommandId); this.registerDetachedTab(result.detachedTab); return result; diff --git a/core/lib/Tab.ts b/core/lib/Tab.ts index 8ea444386..18bdc342d 100644 --- a/core/lib/Tab.ts +++ b/core/lib/Tab.ts @@ -116,7 +116,6 @@ export default class Tab return this.frameEnvironmentsByPuppetId.get(this.puppetPage.mainFrame.id); } - // eslint-disable-next-line @typescript-eslint/member-ordering private constructor( session: Session, puppetPage: IPuppetPage, diff --git a/core/models/DomChangesTable.ts b/core/models/DomChangesTable.ts index 8b218126a..0946b51cc 100644 --- a/core/models/DomChangesTable.ts +++ b/core/models/DomChangesTable.ts @@ -80,6 +80,17 @@ export default class DomChangesTable extends SqliteTable { return query.all(frameId, afterCommandId ?? 0).map(DomChangesTable.inflateRecord); } + public getDomChangesForNavigation( + frameNavigationId: number, + upToEventIndex: number, + ): IDomChangeRecord[] { + const query = this.db.prepare( + `select * from ${this.tableName} where frameNavigationId =? and eventIndex <= ?`, + ); + + return query.all(frameNavigationId, upToEventIndex).map(DomChangesTable.inflateRecord); + } + public getChangesSinceNavigation(navigationId: number): IDomChangeRecord[] { const query = this.db.prepare(`select * from ${this.tableName} where frameNavigationId >= ?`); diff --git a/core/models/FragmentsTable.ts b/core/models/FragmentsTable.ts new file mode 100644 index 000000000..763cb2a93 --- /dev/null +++ b/core/models/FragmentsTable.ts @@ -0,0 +1,48 @@ +import { Database as SqliteDatabase } from 'better-sqlite3'; +import SqliteTable from '@ulixee/commons/lib/SqliteTable'; + +export default class FragmentsTable extends SqliteTable { + private names = new Set(); + constructor(readonly db: SqliteDatabase) { + super( + db, + 'Fragments', + [ + ['name', 'TEXT', 'NOT NULL PRIMARY KEY'], + ['commandId', 'INTEGER'], + ['frameNavigationId', 'INTEGER'], + ['domChangeEventIndex', 'INTEGER'], + ['nodePointerId', 'INTEGER'], + ['nodeType', 'TEXT'], + ['nodePreview', 'TEXT'], + ], + true, + ); + } + + public insert(fragment: IFragment): void { + if (this.names.has(fragment.name)) + throw new Error( + `The provided fragment name (${fragment.name}) must be unique for a session.`, + ); + this.queuePendingInsert([ + fragment.name, + fragment.commandId, + fragment.frameNavigationId, + fragment.domChangeEventIndex, + fragment.nodePointerId, + fragment.nodeType, + fragment.nodePreview, + ]); + } +} + +export interface IFragment { + name: string; + commandId: number; + frameNavigationId: number; + domChangeEventIndex: number; + nodePointerId: number; + nodeType: string; + nodePreview: string; +} diff --git a/core/models/FrameNavigationsTable.ts b/core/models/FrameNavigationsTable.ts index dc0352508..b002b0f14 100644 --- a/core/models/FrameNavigationsTable.ts +++ b/core/models/FrameNavigationsTable.ts @@ -46,6 +46,10 @@ export default class FrameNavigationsTable extends SqliteTable, diff --git a/fullstack/test/fragments.test.ts b/fullstack/test/fragments.test.ts new file mode 100644 index 000000000..715cbe14c --- /dev/null +++ b/fullstack/test/fragments.test.ts @@ -0,0 +1,98 @@ +import { Helpers } from '@ulixee/hero-testing'; +import { ITestKoaServer } from '@ulixee/hero-testing/helpers'; +import Hero, { ConnectionToCore } from '../index'; + +let koaServer: ITestKoaServer; +const sendRequestSpy = jest.spyOn(ConnectionToCore.prototype, 'sendRequest'); +beforeAll(async () => { + koaServer = await Helpers.runKoaServer(); +}); +afterAll(Helpers.afterAll); +afterEach(Helpers.afterEach); + +describe('basic Fragment tests', () => { + it('can create fragments', async () => { + koaServer.get('/fragment-basic', ctx => { + ctx.body = ` + +
test 1
+
+
  • Test 2
+
+ + `; + }); + const hero = await openBrowser(`/fragment-basic`); + const test1Element = await hero.document.querySelector('.test1'); + await test1Element.$extractLater('a'); + await test1Element.nextElementSibling.$extractLater('b'); + + await hero.importFragments(await hero.sessionId); + + expect(hero.getFragment('a')).toBeTruthy(); + await expect(hero.getFragment('a').innerText).resolves.toBe('test 1'); + + expect(hero.getFragment('b')).toBeTruthy(); + await expect(hero.getFragment('b').childElementCount).resolves.toBe(1); + await expect(hero.getFragment('b').querySelectorAll('li').length).resolves.toBe(1); + }); + + it('can prefetch fragment commands', async () => { + koaServer.get('/fragment-learn', ctx => { + ctx.body = ` + +
test 1
+
test 2
+ + `; + }); + { + // run 1 + const hero = await openBrowser(`/fragment-learn`); + const test1Element = await hero.document.querySelector('.test1'); + await test1Element.$extractLater('test1'); + await test1Element.nextElementSibling.$extractLater('test2'); + + sendRequestSpy.mockClear(); + await hero.importFragments(await hero.sessionId); + + expect(hero.getFragment('test1')).toBeTruthy(); + await expect(hero.getFragment('test1').innerText).resolves.toBe('test 1'); + await expect(hero.getFragment('test1').hasAttribute('ready')).resolves.toBe(false); + + expect(hero.getFragment('test2')).toBeTruthy(); + await expect(hero.getFragment('test2').innerText).resolves.toBe('test 2'); + await expect(hero.getFragment('test2').hasAttribute('ready')).resolves.toBe(true); + expect(sendRequestSpy).toHaveBeenCalledTimes(5); + // need to close to record jspaths + await hero.close(); + } + { + // run 1 + const hero = await openBrowser(`/fragment-learn`); + const test1Element = await hero.document.querySelector('.test1'); + await test1Element.$extractLater('test1'); + await test1Element.nextElementSibling.$extractLater('test2'); + + sendRequestSpy.mockClear(); + await hero.importFragments(await hero.sessionId); + + expect(hero.getFragment('test1')).toBeTruthy(); + await expect(hero.getFragment('test1').innerText).resolves.toBe('test 1'); + await expect(hero.getFragment('test1').hasAttribute('ready')).resolves.toBe(false); + + expect(hero.getFragment('test2')).toBeTruthy(); + await expect(hero.getFragment('test2').innerText).resolves.toBe('test 2'); + await expect(hero.getFragment('test2').hasAttribute('ready')).resolves.toBe(true); + expect(sendRequestSpy).toHaveBeenCalledTimes(1); + } + }); +}); + +async function openBrowser(path: string) { + const hero = new Hero(); + Helpers.needsClosing.push(hero); + await hero.goto(`${koaServer.baseUrl}${path}`); + await hero.waitForPaintingStable(); + return hero; +}