feat(core): collect and recreate fragments

ulixee · Jan 17, 2022 · 69db46e · 69db46e
1 parent 327b525
commit 69db46e
Show file tree

Hide file tree

Showing 16 changed files with 397 additions and 86 deletions.
diff --git a/client/lib/CoreFrameEnvironment.ts b/client/lib/CoreFrameEnvironment.ts
@@ -82,6 +82,10 @@ export default class CoreFrameEnvironment {
     return await this.commandQueue.run('FrameEnvironment.createRequest', input, init);
   }
 
+  public async createFragment(name: string, jsPath: IJsPath): Promise<INodePointer> {
+    return await this.commandQueue.run('FrameEnvironment.createFragment', name, jsPath);
+  }
+
   public async getUrl(): Promise<string> {
     return await this.commandQueue.run('FrameEnvironment.getUrl');
   }

diff --git a/client/lib/CoreSession.ts b/client/lib/CoreSession.ts
@@ -16,6 +16,7 @@ import IJsPathEventTarget from '../interfaces/IJsPathEventTarget';
 import ConnectionToCore from '../connections/ConnectionToCore';
 import ICommandCounter from '../interfaces/ICommandCounter';
 import ISessionCreateOptions from '@ulixee/hero-interfaces/ISessionCreateOptions';
+import INodePointer from 'awaited-dom/base/INodePointer';
 
 export default class CoreSession implements IJsPathEventTarget {
   public tabsById = new Map<number, CoreTab>();
@@ -141,27 +142,36 @@ export default class CoreSession implements IJsPathEventTarget {
     };
   }
 
-  // @experimental
-  public async loadFrozenTab(
-    sessionId: string,
-    name: string,
-    atCommandId: number,
-    tabId = 1,
-  ): Promise<{ coreTab: CoreTab; prefetchedJsPaths: IJsPathResult[] }> {
-    const { detachedTab, prefetchedJsPaths } = await this.commandQueue.runOutOfBand<{
-      detachedTab: ISessionMeta;
+  public async loadFragments(sessionId: string): Promise<
+    {
+      name: string;
+      nodePointer: INodePointer;
+      coreTab: CoreTab;
       prefetchedJsPaths: IJsPathResult[];
-    }>('Session.loadFrozenTab', sessionId, name, atCommandId, tabId);
-    const coreTab = new CoreTab(
-      { ...detachedTab, sessionName: this.sessionName },
-      this.connectionToCore,
-      this,
-    );
-    this.frozenTabsById.set(detachedTab.tabId, coreTab);
-    return {
-      coreTab,
-      prefetchedJsPaths,
-    };
+    }[]
+  > {
+    const fragments = await this.commandQueue.run<
+      {
+        name: string;
+        nodePointer: INodePointer;
+        detachedTab: ISessionMeta;
+        prefetchedJsPaths: IJsPathResult[];
+      }[]
+    >('Session.loadAllFragments', sessionId);
+    return fragments.map(fragment => {
+      const coreTab = new CoreTab(
+        { ...fragment.detachedTab, sessionName: this.sessionName },
+        this.connectionToCore,
+        this,
+      );
+      this.frozenTabsById.set(fragment.detachedTab.tabId, coreTab);
+      return {
+        name: fragment.name,
+        nodePointer: fragment.nodePointer,
+        coreTab,
+        prefetchedJsPaths: fragment.prefetchedJsPaths,
+      };
+    });
   }
 
   public async close(force = false): Promise<void> {

diff --git a/client/lib/DomExtender.ts b/client/lib/DomExtender.ts
@@ -30,15 +30,14 @@ interface IBaseExtendNode {
   $isClickable: Promise<boolean>;
   $clearValue(): Promise<void>;
   $click(verification?: IElementInteractVerification): Promise<void>;
+  $extractLater(name: string): Promise<void>;
   $type(...typeInteractions: ITypeInteraction[]): Promise<void>;
   $waitForHidden(options?: { timeoutMs?: number }): Promise<ISuperElement>;
   $waitForVisible(options?: { timeoutMs?: number }): Promise<ISuperElement>;
 }
 
 interface IBaseExtendNodeList {
-  $map<T = any>(
-    iteratorFn: (node: ISuperNode, index: number) => Promise<T>
-  ): Promise<T[]>;
+  $map<T = any>(iteratorFn: (node: ISuperNode, index: number) => Promise<T>): Promise<T[]>;
   $reduce<T = any>(
     iteratorFn: (initial: T, node: ISuperNode) => Promise<T>,
     initial: T,
@@ -66,6 +65,11 @@ const NodeExtensionFns: Partial<IBaseExtendNode> = {
     const coreFrame = await getCoreFrame(this);
     await Interactor.run(coreFrame, [{ click: { element: this, verification } }]);
   },
+  async $extractLater(name: string): Promise<void> {
+    const { awaitedPath, awaitedOptions } = awaitedPathState.getState(this);
+    const coreFrame = await awaitedOptions.coreFrame;
+    await coreFrame.createFragment(name, awaitedPath.toJSON());
+  },
   async $type(...typeInteractions: ITypeInteraction[]): Promise<void> {
     const coreFrame = await getCoreFrame(this);
     await this.$click();
@@ -142,14 +146,17 @@ const NodeListExtensionFns: IBaseExtendNodeList = {
     }
     return newArray;
   },
-  async $reduce<T = any>(iteratorFn: (initial: T, node: ISuperNode) => Promise<T>, initial: T): Promise<T> {
+  async $reduce<T = any>(
+    iteratorFn: (initial: T, node: ISuperNode) => Promise<T>,
+    initial: T,
+  ): Promise<T> {
     const nodes = await this;
     for (const node of nodes) {
       initial = await iteratorFn(initial, node);
     }
     return initial;
-  }
-}
+  },
+};
 
 for (const Item of [SuperElement, SuperNode, SuperHTMLElement, Element, Node, HTMLElement]) {
   for (const [key, value] of Object.entries(NodeExtensionFns)) {

diff --git a/client/lib/Fragment.ts b/client/lib/Fragment.ts
@@ -0,0 +1,37 @@
+import AwaitedPath from 'awaited-dom/base/AwaitedPath';
+import { ISuperElement } from 'awaited-dom/base/interfaces/super';
+import FrozenTab from './FrozenTab';
+import { createInstanceWithNodePointer } from './SetupAwaitedHandler';
+import { getState as getFrozenFrameState } from './FrozenFrameEnvironment';
+import INodePointer from 'awaited-dom/base/INodePointer';
+import StateMachine from 'awaited-dom/base/StateMachine';
+import IAwaitedOptions from '../interfaces/IAwaitedOptions';
+
+const awaitedPathState = StateMachine<
+  any,
+  { awaitedPath: AwaitedPath; awaitedOptions: IAwaitedOptions; nodePointer?: INodePointer }
+>();
+
+export default class Fragment {
+  public element: ISuperElement;
+  public name: string;
+
+  readonly #nodePointer: INodePointer;
+  readonly #frozenTab: FrozenTab;
+
+  constructor(frozenTab: FrozenTab, name: string, nodePointer: INodePointer) {
+    this.name = name;
+    this.#frozenTab = frozenTab;
+    this.#nodePointer = nodePointer;
+    this.element = createInstanceWithNodePointer(
+      awaitedPathState,
+      new AwaitedPath(null),
+      getFrozenFrameState(frozenTab.mainFrameEnvironment),
+      nodePointer,
+    );
+  }
+
+  public close(): Promise<void> {
+    return this.#frozenTab.close();
+  }
+}
diff --git a/client/lib/FrozenFrameEnvironment.ts b/client/lib/FrozenFrameEnvironment.ts
@@ -29,7 +29,7 @@ import CoreFrameEnvironment from './CoreFrameEnvironment';
 import FrozenTab from './FrozenTab';
 import * as AwaitedHandler from './SetupAwaitedHandler';
 
-const { getState, setState } = StateMachine<FrozenFrameEnvironment, IState>();
+export const { getState, setState } = StateMachine<FrozenFrameEnvironment, IState>();
 const awaitedPathState = StateMachine<
   any,
   { awaitedPath: AwaitedPath; awaitedOptions: IAwaitedOptions }

diff --git a/client/lib/Hero.ts b/client/lib/Hero.ts
@@ -65,6 +65,7 @@ import ConnectionManager from './ConnectionManager';
 import './DomExtender';
 import IPageStateDefinitions from '../interfaces/IPageStateDefinitions';
 import IMagicSelectorOptions from '@ulixee/hero-interfaces/IMagicSelectorOptions';
+import Fragment from './Fragment';
 
 export const DefaultOptions = {
   defaultBlockedResourceTypes: [BlockedResourceType.None],
@@ -79,8 +80,6 @@ export type IStateOptions = Omit<ISessionCreateOptions, 'externalIds' | 'session
 
 export interface IState {
   connection: ConnectionManager;
-  isClosing: boolean;
-  options: IStateOptions;
   clientPlugins: IClientPlugin[];
 }
 
@@ -114,10 +113,14 @@ export default class Hero extends AwaitedEventTarget<{
   protected static options: IHeroDefaults = { ...DefaultOptions };
   private static emitter = new EventEmitter();
 
+  readonly #connectManagerIsReady = createPromise();
+  readonly #fragmentsByName = new Map<string, Fragment>();
+  readonly #options: IStateOptions;
+  #isClosing = false;
+
   constructor(options: IHeroCreateOptions = {}) {
-    const connectionManagerIsReady = createPromise();
     super(async () => {
-      await connectionManagerIsReady.promise;
+      await this.#connectManagerIsReady.promise;
       return {
         target: getState(this).connection.getConnectedCoreSessionOrReject(),
       };
@@ -133,7 +136,7 @@ export default class Hero extends AwaitedEventTarget<{
     const sessionName = scriptInstance.generateSessionName(options.name);
     delete options.name;
 
-    options = {
+    this.#options = {
       ...options,
       mode: options.mode ?? scriptInstance.mode,
       sessionName,
@@ -142,16 +145,14 @@ export default class Hero extends AwaitedEventTarget<{
       corePluginPaths: [],
     } as IStateOptions;
 
-    const connection = new ConnectionManager(this, options);
+    const connection = new ConnectionManager(this, this.#options);
 
     setState(this, {
       connection,
-      isClosing: false,
-      options,
       clientPlugins: [],
     });
 
-    connectionManagerIsReady.resolve();
+    this.#connectManagerIsReady.resolve();
   }
 
   public get activeTab(): Tab {
@@ -192,7 +193,7 @@ export default class Hero extends AwaitedEventTarget<{
   }
 
   public get sessionName(): Promise<string> {
-    return Promise.resolve(getState(this).options.sessionName);
+    return Promise.resolve(this.#options.sessionName);
   }
 
   public get meta(): Promise<IHeroMeta> {
@@ -227,9 +228,9 @@ export default class Hero extends AwaitedEventTarget<{
   // METHODS
 
   public async close(): Promise<void> {
-    const { isClosing, connection } = getState(this);
-    if (isClosing) return;
-    setState(this, { isClosing: true });
+    const { connection } = getState(this);
+    if (this.#isClosing) return;
+    this.#isClosing = true;
 
     try {
       return await connection.close();
@@ -243,19 +244,25 @@ export default class Hero extends AwaitedEventTarget<{
     await tab.close();
   }
 
-  // @experimental
-  public loadFrozenTabs(
-    sessionId: string,
-    tabNameToCommandId: { [name: string]: number },
-  ): { [name: string]: FrozenTab } {
-    const coreSession = getState(this).connection.getConnectedCoreSessionOrReject();
+  public getFragment<T extends ISuperElement>(name: string): T {
+    return this.#fragmentsByName.get(name).element as T;
+  }
 
-    const result: { [name: string]: FrozenTab } = {};
-    for (const [name, commandId] of Object.entries(tabNameToCommandId)) {
-      const coreFrozenTab = coreSession.then(x => x.loadFrozenTab(sessionId, name, commandId));
-      result[name] = new FrozenTab(this, coreFrozenTab);
-    }
-    return result;
+  public async importFragments(sessionId: string): Promise<Fragment[]> {
+    const coreSession = await getState(this).connection.getConnectedCoreSessionOrReject();
+    const fragments = await coreSession.loadFragments(sessionId);
+    return fragments.map(x => {
+      const frozenTab = new FrozenTab(
+        this,
+        Promise.resolve({
+          coreTab: x.coreTab,
+          prefetchedJsPaths: x.prefetchedJsPaths,
+        }),
+      );
+      const fragment = new Fragment(frozenTab, x.name, x.nodePointer);
+      this.#fragmentsByName.set(fragment.name, fragment);
+      return fragment;
+    });
   }
 
   public detach(tab: Tab, key?: string): FrozenTab {
@@ -340,7 +347,7 @@ export default class Hero extends AwaitedEventTarget<{
   // PLUGINS
 
   public use(PluginObject: string | IClientPluginClass | { [name: string]: IPluginClass }): void {
-    const { clientPlugins, options, connection } = getState(this);
+    const { clientPlugins, connection } = getState(this);
     const ClientPluginsById: { [id: string]: IClientPluginClass } = {};
 
     if (connection.hasConnected) {
@@ -354,7 +361,7 @@ export default class Hero extends AwaitedEventTarget<{
       const CorePlugins = filterPlugins(Plugins, PluginTypes.CorePlugin);
       const ClientPlugins = filterPlugins<IClientPluginClass>(Plugins, PluginTypes.ClientPlugin);
       if (CorePlugins.length) {
-        options.corePluginPaths.push(PluginObject);
+        this.#options.corePluginPaths.push(PluginObject);
       }
       ClientPlugins.forEach(ClientPlugin => (ClientPluginsById[ClientPlugin.id] = ClientPlugin));
     } else {
@@ -372,7 +379,7 @@ export default class Hero extends AwaitedEventTarget<{
         clientPlugin.onHero(this, connection.sendToActiveTab);
       }
 
-      options.dependencyMap[ClientPlugin.id] = ClientPlugin.coreDependencyIds || [];
+      this.#options.dependencyMap[ClientPlugin.id] = ClientPlugin.coreDependencyIds || [];
     });
   }
 

diff --git a/core/dbs/SessionDb.ts b/core/dbs/SessionDb.ts
@@ -24,6 +24,7 @@ import SocketsTable from '../models/SocketsTable';
 import Core from '../index';
 import StorageChangesTable from '../models/StorageChangesTable';
 import AwaitedEventsTable from '../models/AwaitedEventsTable';
+import FragmentsTable from '../models/FragmentsTable';
 
 const { log } = Log(module);
 
@@ -48,6 +49,7 @@ export default class SessionDb {
   public readonly resourceStates: ResourceStatesTable;
   public readonly websocketMessages: WebsocketMessagesTable;
   public readonly domChanges: DomChangesTable;
+  public readonly fragments: FragmentsTable;
   public readonly pageLogs: PageLogsTable;
   public readonly sessionLogs: SessionLogsTable;
   public readonly session: SessionTable;
@@ -87,6 +89,7 @@ export default class SessionDb {
     this.resourceStates = new ResourceStatesTable(this.db);
     this.websocketMessages = new WebsocketMessagesTable(this.db);
     this.domChanges = new DomChangesTable(this.db);
+    this.fragments = new FragmentsTable(this.db);
     this.pageLogs = new PageLogsTable(this.db);
     this.session = new SessionTable(this.db);
     this.mouseEvents = new MouseEventsTable(this.db);
@@ -108,6 +111,7 @@ export default class SessionDb {
       this.resourceStates,
       this.websocketMessages,
       this.domChanges,
+      this.fragments,
       this.pageLogs,
       this.session,
       this.mouseEvents,