From a44c0db81a1a280f66f296d6c3bb81977f70b80a Mon Sep 17 00:00:00 2001 From: Caleb Clark Date: Fri, 4 Feb 2022 12:31:06 -0600 Subject: [PATCH] feat: added support needed for elem. in databox --- client/interfaces/ICoreSession.ts | 21 ++++++++++ client/lib/CoreFrameEnvironment.ts | 19 +++++----- client/lib/CoreSession.ts | 51 +++++++++++++++---------- client/lib/CoreTab.ts | 2 +- client/lib/Hero.ts | 61 +----------------------------- client/lib/Resource.ts | 5 ++- client/lib/WebsocketResource.ts | 3 +- client/package.json | 3 +- core/lib/FrameEnvironment.ts | 6 +-- core/lib/Tab.ts | 6 ++- 10 files changed, 78 insertions(+), 99 deletions(-) create mode 100644 client/interfaces/ICoreSession.ts diff --git a/client/interfaces/ICoreSession.ts b/client/interfaces/ICoreSession.ts new file mode 100644 index 000000000..73a2ed222 --- /dev/null +++ b/client/interfaces/ICoreSession.ts @@ -0,0 +1,21 @@ +import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet'; +import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement'; +import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource'; + +// This interface exists for DataboxInternal to import + +export default interface ICoreSession { + sessionId: string; + collectSnippet(name: string, value: any): Promise; + getCollectedSnippets(sessionId: string, name: string): Promise; + getCollectedElements(sessionId: string, name: string): Promise; + getCollectedResources(sessionId: string, name: string): Promise; + recordOutput(changes: IOutputChangeToRecord[]): void +} + +export interface IOutputChangeToRecord { + type: string; + value: any; + path: string; + timestamp: number; +} diff --git a/client/lib/CoreFrameEnvironment.ts b/client/lib/CoreFrameEnvironment.ts index 82390a001..fc9c741b5 100644 --- a/client/lib/CoreFrameEnvironment.ts +++ b/client/lib/CoreFrameEnvironment.ts @@ -14,20 +14,21 @@ import { } from '@ulixee/hero-interfaces/jsPathFnNames'; import IWaitForOptions from '@ulixee/hero-interfaces/IWaitForOptions'; import IFrameMeta from '@ulixee/hero-interfaces/IFrameMeta'; -import CoreCommandQueue from './CoreCommandQueue'; import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta'; import { INodeVisibility } from '@ulixee/hero-interfaces/INodeVisibility'; +import StateMachine from 'awaited-dom/base/StateMachine'; +import { IElementIsolate, INodeIsolate } from 'awaited-dom/base/interfaces/isolate'; +import { ISuperElement } from 'awaited-dom/base/interfaces/super'; +import TimeoutError from '@ulixee/commons/interfaces/TimeoutError'; +import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement'; +import IAwaitedOptions from '../interfaces/IAwaitedOptions'; +import CoreCommandQueue from './CoreCommandQueue'; +import CoreTab from './CoreTab'; import { convertJsPathArgs, createInstanceWithNodePointer, delegate as AwaitedHandler, } from './SetupAwaitedHandler'; -import StateMachine from 'awaited-dom/base/StateMachine'; -import IAwaitedOptions from '../interfaces/IAwaitedOptions'; -import { IElementIsolate, INodeIsolate } from 'awaited-dom/base/interfaces/isolate'; -import CoreTab from './CoreTab'; -import { ISuperElement } from 'awaited-dom/base/interfaces/super'; -import TimeoutError from '@ulixee/commons/interfaces/TimeoutError'; const awaitedPathState = StateMachine< any, @@ -94,8 +95,8 @@ export default class CoreFrameEnvironment { return await this.commandQueue.run('FrameEnvironment.createRequest', input, init); } - public async collectElement(name: string, jsPath: IJsPath): Promise { - await this.commandQueue.run('FrameEnvironment.collectElement', name, jsPath); + public async collectElement(name: string, jsPath: IJsPath, waitForElement = false): Promise { + return await this.commandQueue.run('FrameEnvironment.collectElement', name, jsPath, waitForElement); } public async getUrl(): Promise { diff --git a/client/lib/CoreSession.ts b/client/lib/CoreSession.ts index 533890167..43e6dbf11 100644 --- a/client/lib/CoreSession.ts +++ b/client/lib/CoreSession.ts @@ -19,6 +19,8 @@ import ISessionCreateOptions from '@ulixee/hero-interfaces/ISessionCreateOptions import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta'; import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement'; import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet'; +import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource'; +import { IOutputChangeToRecord } from '../interfaces/ICoreSession'; export default class CoreSession implements IJsPathEventTarget { public tabsById = new Map(); @@ -50,7 +52,6 @@ export default class CoreSession implements IJsPathEventTarget { private cliPrompt: ReadLine; private isClosing = false; private shutdownPromise: Promise<{ didKeepAlive: boolean; message?: string }>; - private extractorPromises: Promise[] = []; constructor( sessionMeta: ISessionMeta & { sessionName: string }, @@ -99,15 +100,6 @@ export default class CoreSession implements IJsPathEventTarget { return this.commandQueue.run('Session.getHeroMeta'); } - public recordOutput( - changes: { type: string; value: any; path: string; timestamp: Date }[], - ): void { - for (const change of changes as any[]) { - change.lastCommandId = this.lastCommandId; - } - this.commandQueue.record({ command: 'Session.recordOutput', args: changes }); - } - public async configure(options?: Partial): Promise { await this.commandQueue.run('Session.configure', options); } @@ -154,12 +146,13 @@ export default class CoreSession implements IJsPathEventTarget { }; } - public addExtractorPromises(promise: Promise): void { - this.extractorPromises.push(promise); - } + // START OF PRIVATE APIS FOR DATABOX ///////////////////////////////////////////////////////////// - public getExtractorPromises(): Promise[] { - return [...this.extractorPromises]; + public recordOutput(changes: IOutputChangeToRecord[]): void { + for (const change of changes as any[]) { + change.lastCommandId = this.lastCommandId; + } + this.commandQueue.record({ command: 'Session.recordOutput', args: changes }); } public async collectSnippet(name: string, value: any): Promise { @@ -170,14 +163,34 @@ export default class CoreSession implements IJsPathEventTarget { return await this.commandQueue.run('Session.getCollectedSnippets', sessionId, name); } - public async getCollectedResources(sessionId: string, name: string): Promise { - return await this.commandQueue.run('Session.getCollectedResources', sessionId, name); - } - public async getCollectedElements(sessionId: string, name: string): Promise { return await this.commandQueue.run('Session.getCollectedElements', sessionId, name); } + public async getCollectedResources(sessionId: string, name: string): Promise { + const resources: IResourceMeta[] = await this.commandQueue.run('Session.getCollectedResources', sessionId, name); + const results: ICollectedResource[] = []; + for (const resource of resources) { + const buffer = resource.response?.body; + delete resource.response?.body; + + const properties: PropertyDescriptorMap = { + buffer: { get: () => buffer, enumerable: true }, + json: { get: () => (buffer ? JSON.parse(buffer.toString()) : null), enumerable: true }, + text: { get: () => buffer?.toString(), enumerable: true }, + }; + + if (resource.response) { + Object.defineProperties(resource.response, properties); + } + Object.defineProperties(resource, properties); + results.push(resource as ICollectedResource); + } + return results; + } + + // END OF PRIVATE APIS FOR DATABOX /////////////////////////////////////////////////////////////// + public async close(force = false): Promise { await this.shutdownPromise; if (this.isClosing) return; diff --git a/client/lib/CoreTab.ts b/client/lib/CoreTab.ts index 233190900..d0438d4da 100644 --- a/client/lib/CoreTab.ts +++ b/client/lib/CoreTab.ts @@ -32,6 +32,7 @@ export default class CoreTab implements IJsPathEventTarget { public sessionId: string; public commandQueue: CoreCommandQueue; public eventHeap: CoreEventHeap; + public readonly coreSession: CoreSession; public get mainFrameEnvironment(): CoreFrameEnvironment { return this.frameEnvironmentsById.get(this.mainFrameId); } @@ -41,7 +42,6 @@ export default class CoreTab implements IJsPathEventTarget { private readonly flowHandlers: IFlowHandler[] = []; private readonly connection: ConnectionToCore; private readonly mainFrameId: number; - private readonly coreSession: CoreSession; constructor( meta: ISessionMeta & { sessionName: string }, diff --git a/client/lib/Hero.ts b/client/lib/Hero.ts index e9461e479..3d2136c74 100644 --- a/client/lib/Hero.ts +++ b/client/lib/Hero.ts @@ -56,16 +56,13 @@ import FrameEnvironment, { getCoreFrameEnvironmentForPosition } from './FrameEnv import FrozenTab from './FrozenTab'; import FileChooser from './FileChooser'; import CoreFrameEnvironment from './CoreFrameEnvironment'; -import './DomExtender'; -import ICollectedResource from '@ulixee/hero-interfaces/ICollectedResource'; -import ICollectedElement from '@ulixee/hero-interfaces/ICollectedElement'; import IDomState, { IDomStateAllFn } from '@ulixee/hero-interfaces/IDomState'; import DomState from './DomState'; import ConnectionToCore from '../connections/ConnectionToCore'; import CoreSession from './CoreSession'; import { InternalPropertiesSymbol } from './InternalProperties'; import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties'; -import ICollectedSnippet from '@ulixee/hero-interfaces/ICollectedSnippet'; +import './DomExtender'; export const DefaultOptions = { defaultBlockedResourceTypes: [BlockedResourceType.None], @@ -230,11 +227,6 @@ export default class Hero extends AwaitedEventTarget<{ // METHODS - public async recordOutput(changesToRecord): Promise { - const coreSession = await this.#getCoreSessionOrReject(); - coreSession.recordOutput(changesToRecord); - } - public close(): Promise { return (this.#isClosingPromise ??= new Promise(async (resolve, reject) => { try { @@ -269,57 +261,6 @@ export default class Hero extends AwaitedEventTarget<{ return await this.activeTab.findResource(filter, options); } - public async collectSnippet(name: string, value: any): Promise { - const coreSession = await this.#getCoreSessionOrReject(); - await coreSession.collectSnippet(name, value); - } - - public async getCollectedSnippets( - sessionIdPromise: Promise, - name: string, - ): Promise { - const sessionId = await sessionIdPromise; - const coreSession = await this.#getCoreSessionOrReject(); - return await coreSession.getCollectedSnippets(sessionId, name); - } - - public async getCollectedResources( - sessionIdPromise: Promise, - name: string, - ): Promise { - const sessionId = await sessionIdPromise; - const coreSession = await this.#getCoreSessionOrReject(); - const resources = await coreSession.getCollectedResources(sessionId, name); - - const results: ICollectedResource[] = []; - for (const resource of resources) { - const buffer = resource.response?.body; - delete resource.response?.body; - - const properties: PropertyDescriptorMap = { - buffer: { get: () => buffer, enumerable: true }, - json: { get: () => (buffer ? JSON.parse(buffer.toString()) : null), enumerable: true }, - text: { get: () => buffer?.toString(), enumerable: true }, - }; - - if (resource.response) { - Object.defineProperties(resource.response, properties); - } - Object.defineProperties(resource, properties); - results.push(resource as ICollectedResource); - } - return results; - } - - public async getCollectedElements( - sessionIdPromise: Promise, - name: string, - ): Promise { - const sessionId = await sessionIdPromise; - const coreSession = await this.#getCoreSessionOrReject(); - return await coreSession.getCollectedElements(sessionId, name); - } - public detach(tab: Tab, key?: string): FrozenTab { const callsitePath = JSON.stringify(getCallSite(module.filename, scriptInstance.entrypoint)); diff --git a/client/lib/Resource.ts b/client/lib/Resource.ts index 3cf018920..8b77546c1 100644 --- a/client/lib/Resource.ts +++ b/client/lib/Resource.ts @@ -4,14 +4,15 @@ import IResourceMeta from '@ulixee/hero-interfaces/IResourceMeta'; import Timer from '@ulixee/commons/lib/Timer'; import IWaitForResourceOptions from '@ulixee/hero-interfaces/IWaitForResourceOptions'; import TimeoutError from '@ulixee/commons/interfaces/TimeoutError'; +import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties'; import * as Util from 'util'; import CoreTab from './CoreTab'; import ResourceRequest, { createResourceRequest } from './ResourceRequest'; import ResourceResponse, { createResourceResponse } from './ResourceResponse'; import { createWebsocketResource } from './WebsocketResource'; import IWaitForResourceFilter from '../interfaces/IWaitForResourceFilter'; +import { InternalPropertiesSymbol } from './InternalProperties'; import Tab, { getCoreTab } from './Tab'; -import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterProperties'; const propertyKeys: (keyof Resource)[] = [ 'url', @@ -31,7 +32,7 @@ export default class Resource { readonly request: ResourceRequest; readonly response: ResourceResponse; - get [Symbol.for('@ulixee/internalState')](): { + get [InternalPropertiesSymbol](): { coreTabPromise: Promise; resourceMeta: IResourceMeta; } { diff --git a/client/lib/WebsocketResource.ts b/client/lib/WebsocketResource.ts index 2d962c889..19cc0e8f9 100644 --- a/client/lib/WebsocketResource.ts +++ b/client/lib/WebsocketResource.ts @@ -8,6 +8,7 @@ import CoreTab from './CoreTab'; import ResourceRequest, { createResourceRequest } from './ResourceRequest'; import ResourceResponse, { createResourceResponse } from './ResourceResponse'; import AwaitedEventTarget from './AwaitedEventTarget'; +import { InternalPropertiesSymbol } from './InternalProperties'; interface IEventType { message: (message: IWebsocketMessage) => void; @@ -32,7 +33,7 @@ export default class WebsocketResource extends AwaitedEventTarget { readonly request: ResourceRequest; readonly response: ResourceResponse; - get [Symbol.for('@ulixee/internalState')](): { + get [InternalPropertiesSymbol](): { coreTabPromise: Promise; resourceMeta: IResourceMeta; } { diff --git a/client/package.json b/client/package.json index 5363a60e0..362fee84e 100644 --- a/client/package.json +++ b/client/package.json @@ -7,7 +7,8 @@ "import": "./index.mjs", "require": "./index.cjs" }, - "./lib/DomExtender": "./lib/DomExtender.js" + "./lib/DomExtender": "./lib/DomExtender.js", + "./lib/InternalProperties": "./lib/InternalProperties.js" }, "dependencies": { "@ulixee/commons": "1.5.9", diff --git a/core/lib/FrameEnvironment.ts b/core/lib/FrameEnvironment.ts index 1175446f9..e4139fb04 100644 --- a/core/lib/FrameEnvironment.ts +++ b/core/lib/FrameEnvironment.ts @@ -264,7 +264,6 @@ export default class FrameEnvironment const domChangesTimestamp = this.lastDomChangeTimestamp; const elements: ICollectedElement[] = []; - if (nodePointer.iterableItems && nodePointer.iterableIsState) { for (const item of nodePointer.iterableItems as INodePointer[]) { elements.push({ @@ -279,7 +278,7 @@ export default class FrameEnvironment domChangesTimestamp, }); } - } else { + } else if (!nodePointer.iterableItems) { elements.push({ name, nodePointerId: nodePointer.id, @@ -300,8 +299,7 @@ export default class FrameEnvironment promises.push(elementHtmlPromise); } } - await Promise.all(promises); - return elements; + return waitForElement ? await Promise.all(promises) : elements; } public async execJsPath(jsPath: IJsPath): Promise> { diff --git a/core/lib/Tab.ts b/core/lib/Tab.ts index 803b04ad0..97beddf3c 100644 --- a/core/lib/Tab.ts +++ b/core/lib/Tab.ts @@ -537,18 +537,20 @@ export default class Tab this.mirrorNetwork.addResource(resourceSummary); } - public onElementRequested(collectedElement: ICollectedElement): Promise { + public async onElementRequested(collectedElement: ICollectedElement): Promise { const resolvable = new Resolvable(); const resolveExisting = Promise.all(this.collectedElementsPendingHTML); this.collectedElementsPendingHTML.add(resolvable); this.session.db.collectedElements.insert(collectedElement); - return resolveExisting + await resolveExisting .then(() => this.getElementHtml(collectedElement)) .then(resolvable.resolve) .catch(resolvable.resolve) .finally(() => this.collectedElementsPendingHTML.delete(resolvable)); + + return resolvable; } public async getElementHtml(collectedElement: ICollectedElement): Promise {