feat: added support needed for elem. in databox

ulixee · Feb 4, 2022 · a44c0db · a44c0db
1 parent 494b297
commit a44c0db
Show file tree

Hide file tree

Showing 10 changed files with 78 additions and 99 deletions.
diff --git a/client/interfaces/ICoreSession.ts b/client/interfaces/ICoreSession.ts
@@ -0,0 +1,21 @@
+import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet';
+import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement';
+import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource';
+
+// This interface exists for DataboxInternal to import
+
+export default interface ICoreSession {
+  sessionId: string;
+  collectSnippet(name: string, value: any): Promise<void>;
+  getCollectedSnippets(sessionId: string, name: string): Promise<ICollectedSnippet[]>;
+  getCollectedElements(sessionId: string, name: string): Promise<ICollectedElement[]>;
+  getCollectedResources(sessionId: string, name: string): Promise<ICollectedResource[]>;
+  recordOutput(changes: IOutputChangeToRecord[]): void
+}
+
+export interface IOutputChangeToRecord {
+  type: string;
+  value: any;
+  path: string;
+  timestamp: number;
+}
diff --git a/client/lib/CoreFrameEnvironment.ts b/client/lib/CoreFrameEnvironment.ts
@@ -14,20 +14,21 @@ import {
 } from '@ulixee/hero-interfaces/jsPathFnNames';
 import IWaitForOptions from '@ulixee/hero-interfaces/IWaitForOptions';
 import IFrameMeta from '@ulixee/hero-interfaces/IFrameMeta';
-import CoreCommandQueue from './CoreCommandQueue';
 import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta';
 import { INodeVisibility } from '@ulixee/hero-interfaces/INodeVisibility';
+import StateMachine from 'awaited-dom/base/StateMachine';
+import { IElementIsolate, INodeIsolate } from 'awaited-dom/base/interfaces/isolate';
+import { ISuperElement } from 'awaited-dom/base/interfaces/super';
+import TimeoutError from '@ulixee/commons/interfaces/TimeoutError';
+import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement';
+import IAwaitedOptions from '../interfaces/IAwaitedOptions';
+import CoreCommandQueue from './CoreCommandQueue';
+import CoreTab from './CoreTab';
 import {
   convertJsPathArgs,
   createInstanceWithNodePointer,
   delegate as AwaitedHandler,
 } from './SetupAwaitedHandler';
-import StateMachine from 'awaited-dom/base/StateMachine';
-import IAwaitedOptions from '../interfaces/IAwaitedOptions';
-import { IElementIsolate, INodeIsolate } from 'awaited-dom/base/interfaces/isolate';
-import CoreTab from './CoreTab';
-import { ISuperElement } from 'awaited-dom/base/interfaces/super';
-import TimeoutError from '@ulixee/commons/interfaces/TimeoutError';
 
 const awaitedPathState = StateMachine<
   any,
@@ -94,8 +95,8 @@ export default class CoreFrameEnvironment {
     return await this.commandQueue.run('FrameEnvironment.createRequest', input, init);
   }
 
-  public async collectElement(name: string, jsPath: IJsPath): Promise<void> {
-    await this.commandQueue.run('FrameEnvironment.collectElement', name, jsPath);
+  public async collectElement(name: string, jsPath: IJsPath, waitForElement = false): Promise<ICollectedElement[]> {
+    return await this.commandQueue.run('FrameEnvironment.collectElement', name, jsPath, waitForElement);
   }
 
   public async getUrl(): Promise<string> {

diff --git a/client/lib/CoreSession.ts b/client/lib/CoreSession.ts
@@ -19,6 +19,8 @@ import ISessionCreateOptions from '@ulixee/hero-interfaces/ISessionCreateOptions
 import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta';
 import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement';
 import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet';
+import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource';
+import { IOutputChangeToRecord } from '../interfaces/ICoreSession';
 
 export default class CoreSession implements IJsPathEventTarget {
   public tabsById = new Map<number, CoreTab>();
@@ -50,7 +52,6 @@ export default class CoreSession implements IJsPathEventTarget {
   private cliPrompt: ReadLine;
   private isClosing = false;
   private shutdownPromise: Promise<{ didKeepAlive: boolean; message?: string }>;
-  private extractorPromises: Promise<void>[] = [];
 
   constructor(
     sessionMeta: ISessionMeta & { sessionName: string },
@@ -99,15 +100,6 @@ export default class CoreSession implements IJsPathEventTarget {
     return this.commandQueue.run('Session.getHeroMeta');
   }
 
-  public recordOutput(
-    changes: { type: string; value: any; path: string; timestamp: Date }[],
-  ): void {
-    for (const change of changes as any[]) {
-      change.lastCommandId = this.lastCommandId;
-    }
-    this.commandQueue.record({ command: 'Session.recordOutput', args: changes });
-  }
-
   public async configure(options?: Partial<IConfigureSessionOptions>): Promise<void> {
     await this.commandQueue.run('Session.configure', options);
   }
@@ -154,12 +146,13 @@ export default class CoreSession implements IJsPathEventTarget {
     };
   }
 
-  public addExtractorPromises(promise: Promise<void>): void {
-    this.extractorPromises.push(promise);
-  }
+  // START OF PRIVATE APIS FOR DATABOX /////////////////////////////////////////////////////////////
 
-  public getExtractorPromises(): Promise<void>[] {
-    return [...this.extractorPromises];
+  public recordOutput(changes: IOutputChangeToRecord[]): void {
+    for (const change of changes as any[]) {
+      change.lastCommandId = this.lastCommandId;
+    }
+    this.commandQueue.record({ command: 'Session.recordOutput', args: changes });
   }
 
   public async collectSnippet(name: string, value: any): Promise<void> {
@@ -170,14 +163,34 @@ export default class CoreSession implements IJsPathEventTarget {
     return await this.commandQueue.run('Session.getCollectedSnippets', sessionId, name);
   }
 
-  public async getCollectedResources(sessionId: string, name: string): Promise<IResourceMeta[]> {
-    return await this.commandQueue.run('Session.getCollectedResources', sessionId, name);
-  }
-
   public async getCollectedElements(sessionId: string, name: string): Promise<ICollectedElement[]> {
     return await this.commandQueue.run('Session.getCollectedElements', sessionId, name);
   }
 
+  public async getCollectedResources(sessionId: string, name: string): Promise<ICollectedResource[]> {
+    const resources: IResourceMeta[] = await this.commandQueue.run('Session.getCollectedResources', sessionId, name);
+    const results: ICollectedResource[] = [];
+    for (const resource of resources) {
+      const buffer = resource.response?.body;
+      delete resource.response?.body;
+
+      const properties: PropertyDescriptorMap = {
+        buffer: { get: () => buffer, enumerable: true },
+        json: { get: () => (buffer ? JSON.parse(buffer.toString()) : null), enumerable: true },
+        text: { get: () => buffer?.toString(), enumerable: true },
+      };
+
+      if (resource.response) {
+        Object.defineProperties(resource.response, properties);
+      }
+      Object.defineProperties(resource, properties);
+      results.push(resource as ICollectedResource);
+    }
+    return results;
+  }
+
+  // END OF PRIVATE APIS FOR DATABOX ///////////////////////////////////////////////////////////////
+
   public async close(force = false): Promise<void> {
     await this.shutdownPromise;
     if (this.isClosing) return;

diff --git a/client/lib/CoreTab.ts b/client/lib/CoreTab.ts
@@ -32,6 +32,7 @@ export default class CoreTab implements IJsPathEventTarget {
   public sessionId: string;
   public commandQueue: CoreCommandQueue;
   public eventHeap: CoreEventHeap;
+  public readonly coreSession: CoreSession;
   public get mainFrameEnvironment(): CoreFrameEnvironment {
     return this.frameEnvironmentsById.get(this.mainFrameId);
   }
@@ -41,7 +42,6 @@ export default class CoreTab implements IJsPathEventTarget {
   private readonly flowHandlers: IFlowHandler[] = [];
   private readonly connection: ConnectionToCore;
   private readonly mainFrameId: number;
-  private readonly coreSession: CoreSession;
 
   constructor(
     meta: ISessionMeta & { sessionName: string },

diff --git a/client/lib/Hero.ts b/client/lib/Hero.ts
@@ -56,16 +56,13 @@ import FrameEnvironment, { getCoreFrameEnvironmentForPosition } from './FrameEnv
 import FrozenTab from './FrozenTab';
 import FileChooser from './FileChooser';
 import CoreFrameEnvironment from './CoreFrameEnvironment';
-import './DomExtender';
-import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource';
-import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement';
 import IDomState, { IDomStateAllFn } from '@ulixee/hero-interfaces/IDomState';
 import DomState from './DomState';
 import ConnectionToCore from '../connections/ConnectionToCore';
 import CoreSession from './CoreSession';
 import { InternalPropertiesSymbol } from './InternalProperties';
 import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties';
-import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet';
+import './DomExtender';
 
 export const DefaultOptions = {
   defaultBlockedResourceTypes: [BlockedResourceType.None],
@@ -230,11 +227,6 @@ export default class Hero extends AwaitedEventTarget<{
 
   // METHODS
 
-  public async recordOutput(changesToRecord): Promise<void> {
-    const coreSession = await this.#getCoreSessionOrReject();
-    coreSession.recordOutput(changesToRecord);
-  }
-
   public close(): Promise<void> {
     return (this.#isClosingPromise ??= new Promise(async (resolve, reject) => {
       try {
@@ -269,57 +261,6 @@ export default class Hero extends AwaitedEventTarget<{
     return await this.activeTab.findResource(filter, options);
   }
 
-  public async collectSnippet(name: string, value: any): Promise<void> {
-    const coreSession = await this.#getCoreSessionOrReject();
-    await coreSession.collectSnippet(name, value);
-  }
-
-  public async getCollectedSnippets(
-    sessionIdPromise: Promise<string>,
-    name: string,
-  ): Promise<ICollectedSnippet[]> {
-    const sessionId = await sessionIdPromise;
-    const coreSession = await this.#getCoreSessionOrReject();
-    return await coreSession.getCollectedSnippets(sessionId, name);
-  }
-
-  public async getCollectedResources(
-    sessionIdPromise: Promise<string>,
-    name: string,
-  ): Promise<ICollectedResource[]> {
-    const sessionId = await sessionIdPromise;
-    const coreSession = await this.#getCoreSessionOrReject();
-    const resources = await coreSession.getCollectedResources(sessionId, name);
-
-    const results: ICollectedResource[] = [];
-    for (const resource of resources) {
-      const buffer = resource.response?.body;
-      delete resource.response?.body;
-
-      const properties: PropertyDescriptorMap = {
-        buffer: { get: () => buffer, enumerable: true },
-        json: { get: () => (buffer ? JSON.parse(buffer.toString()) : null), enumerable: true },
-        text: { get: () => buffer?.toString(), enumerable: true },
-      };
-
-      if (resource.response) {
-        Object.defineProperties(resource.response, properties);
-      }
-      Object.defineProperties(resource, properties);
-      results.push(resource as ICollectedResource);
-    }
-    return results;
-  }
-
-  public async getCollectedElements(
-    sessionIdPromise: Promise<string>,
-    name: string,
-  ): Promise<ICollectedElement[]> {
-    const sessionId = await sessionIdPromise;
-    const coreSession = await this.#getCoreSessionOrReject();
-    return await coreSession.getCollectedElements(sessionId, name);
-  }
-
   public detach(tab: Tab, key?: string): FrozenTab {
     const callsitePath = JSON.stringify(getCallSite(module.filename, scriptInstance.entrypoint));
 

diff --git a/client/lib/Resource.ts b/client/lib/Resource.ts
@@ -4,14 +4,15 @@ import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta';
 import Timer from '@ulixee/commons/lib/Timer';
 import IWaitForResourceOptions from '@ulixee/hero-interfaces/IWaitForResourceOptions';
 import TimeoutError from '@ulixee/commons/interfaces/TimeoutError';
+import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties';
 import * as Util from 'util';
 import CoreTab from './CoreTab';
 import ResourceRequest, { createResourceRequest } from './ResourceRequest';
 import ResourceResponse, { createResourceResponse } from './ResourceResponse';
 import { createWebsocketResource } from './WebsocketResource';
 import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter';
+import { InternalPropertiesSymbol } from './InternalProperties';
 import Tab, { getCoreTab } from './Tab';
-import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties';
 
 const propertyKeys: (keyof Resource)[] = [
   'url',
@@ -31,7 +32,7 @@ export default class Resource {
   readonly request: ResourceRequest;
   readonly response: ResourceResponse;
 
-  get [Symbol.for('@ulixee/internalState')](): {
+  get [InternalPropertiesSymbol](): {
     coreTabPromise: Promise<CoreTab>;
     resourceMeta: IResourceMeta;
   } {

diff --git a/client/lib/WebsocketResource.ts b/client/lib/WebsocketResource.ts
@@ -8,6 +8,7 @@ import CoreTab from './CoreTab';
 import ResourceRequest, { createResourceRequest } from './ResourceRequest';
 import ResourceResponse, { createResourceResponse } from './ResourceResponse';
 import AwaitedEventTarget from './AwaitedEventTarget';
+import { InternalPropertiesSymbol } from './InternalProperties';
 
 interface IEventType {
   message: (message: IWebsocketMessage) => void;
@@ -32,7 +33,7 @@ export default class WebsocketResource extends AwaitedEventTarget<IEventType> {
   readonly request: ResourceRequest;
   readonly response: ResourceResponse;
 
-  get [Symbol.for('@ulixee/internalState')](): {
+  get [InternalPropertiesSymbol](): {
     coreTabPromise: Promise<CoreTab>;
     resourceMeta: IResourceMeta;
   } {

diff --git a/client/package.json b/client/package.json
@@ -7,7 +7,8 @@
       "import": "./index.mjs",
       "require": "./index.cjs"
     },
-    "./lib/DomExtender": "./lib/DomExtender.js"
+    "./lib/DomExtender": "./lib/DomExtender.js",
+    "./lib/InternalProperties": "./lib/InternalProperties.js"
   },
   "dependencies": {
     "@ulixee/commons": "1.5.9",

diff --git a/core/lib/FrameEnvironment.ts b/core/lib/FrameEnvironment.ts
@@ -264,7 +264,6 @@ export default class FrameEnvironment
     const domChangesTimestamp = this.lastDomChangeTimestamp;
 
     const elements: ICollectedElement[] = [];
-
     if (nodePointer.iterableItems && nodePointer.iterableIsState) {
       for (const item of nodePointer.iterableItems as INodePointer[]) {
         elements.push({
@@ -279,7 +278,7 @@ export default class FrameEnvironment
           domChangesTimestamp,
         });
       }
-    } else {
+    } else if (!nodePointer.iterableItems) {
       elements.push({
         name,
         nodePointerId: nodePointer.id,
@@ -300,8 +299,7 @@ export default class FrameEnvironment
         promises.push(elementHtmlPromise);
       }
     }
-    await Promise.all(promises);
-    return elements;
+    return waitForElement ? await Promise.all(promises) : elements;
   }
 
   public async execJsPath<T>(jsPath: IJsPath): Promise<IExecJsPathResult<T>> {

diff --git a/core/lib/Tab.ts b/core/lib/Tab.ts
@@ -537,18 +537,20 @@ export default class Tab
     this.mirrorNetwork.addResource(resourceSummary);
   }
 
-  public onElementRequested(collectedElement: ICollectedElement): Promise<void> {
+  public async onElementRequested(collectedElement: ICollectedElement): Promise<ICollectedElement> {
     const resolvable = new Resolvable<ICollectedElement>();
     const resolveExisting = Promise.all(this.collectedElementsPendingHTML);
     this.collectedElementsPendingHTML.add(resolvable);
 
     this.session.db.collectedElements.insert(collectedElement);
 
-    return resolveExisting
+    await resolveExisting
       .then(() => this.getElementHtml(collectedElement))
       .then(resolvable.resolve)
       .catch(resolvable.resolve)
       .finally(() => this.collectedElementsPendingHTML.delete(resolvable));
+
+    return resolvable;
   }
 
   public async getElementHtml(collectedElement: ICollectedElement): Promise<ICollectedElement> {