fix(fullstack): queue collect multiple elements

ulixee · Feb 4, 2022 · 95bea01 · 95bea01
1 parent 7d2faa5
commit 95bea01
Show file tree

Hide file tree

Showing 4 changed files with 185 additions and 16 deletions.
diff --git a/core/lib/Tab.ts b/core/lib/Tab.ts
@@ -562,9 +562,9 @@ export default class Tab
         this.sessionId,
         this.session.viewport,
       );
-      await this.mirrorPage.load(paintIndex);
       const frameDomNodeId = this.frameEnvironmentsById.get(collectedElement.frameId).domNodeId;
       collectedElement.outerHTML = await this.mirrorPage.getNodeOuterHtml(
+        paintIndex,
         collectedElement.nodePointerId,
         frameDomNodeId,
       );

diff --git a/fullstack/test/collectElements.test.ts b/fullstack/test/collectElements.test.ts
@@ -0,0 +1,118 @@
+import { Helpers } from '@ulixee/hero-testing';
+import Hero from '..';
+import { InternalPropertiesSymbol } from '@ulixee/hero/lib/InternalProperties';
+import { awaitedPathState } from '@ulixee/hero/lib/DomExtender';
+import { ISuperElement, ISuperNodeList } from 'awaited-dom/base/interfaces/super';
+import CoreSession from '@ulixee/hero/lib/CoreSession';
+
+let koaServer: Helpers.ITestKoaServer;
+beforeAll(async () => {
+  koaServer = await Helpers.runKoaServer();
+});
+afterAll(Helpers.afterAll);
+afterEach(Helpers.afterEach);
+
+describe('basic collect Element tests', () => {
+  it('can extract elements', async () => {
+    koaServer.get('/element-basic', ctx => {
+      ctx.body = `
+        <body>
+          <div class="test1">test 1</div>
+          <div class="test2">
+            <ul><li>Test 2</li></ul>
+          </div>
+        </body>
+      `;
+    });
+    const [hero, coreSession] = await openBrowser(`/element-basic`);
+    const sessionId = await hero.sessionId;
+    const test1Element = await hero.document.querySelector('.test1');
+    await collectElement(test1Element, 'a');
+    await collectElement(test1Element.nextElementSibling, 'b');
+
+    const elementsA = await coreSession.getCollectedElements(sessionId, 'a');
+    expect(elementsA).toHaveLength(1);
+    expect(elementsA[0].outerHTML).toBe('<div class="test1">test 1</div>');
+
+    const elementsB = await coreSession.getCollectedElements(sessionId, 'b');
+    expect(elementsB[0].outerHTML).toBe(`<div class="test2">
+            <ul><li>Test 2</li></ul>
+          </div>`);
+  });
+
+  it('can extract selectorAll lists', async () => {
+    koaServer.get('/element-list', ctx => {
+      ctx.body = `
+        <body>
+
+            <ul>
+              <li class="valid">Test 1</li>
+              <li class="invalid">Test 2</li>
+              <li class="invalid">Test 3</li>
+              <li class="valid">Test 4</li>
+              <li class="valid">Test 5</li>
+            </ul>
+        </body>
+      `;
+    });
+    const [hero, coreSession] = await openBrowser(`/element-list`);
+    const sessionId = await hero.sessionId;
+    await collectElement(hero.document.querySelectorAll('.valid'), 'valid');
+
+    const valid = await coreSession.getCollectedElements(sessionId, 'valid');
+    expect(valid).toHaveLength(3);
+    expect(valid[0].outerHTML).toBe('<li class="valid">Test 1</li>');
+    expect(valid[1].outerHTML).toBe('<li class="valid">Test 4</li>');
+    expect(valid[2].outerHTML).toBe('<li class="valid">Test 5</li>');
+  });
+
+  it('can extract elements across timestamps', async () => {
+    koaServer.get('/growing-list', ctx => {
+      ctx.body = `
+        <body>
+
+            <ul>
+            </ul>
+            <script>
+            
+            function add(text) {
+              const li = document.createElement('LI');
+              li.textContent = text;
+              document.querySelector('ul').appendChild(li);  
+            }
+</script>
+        </body>
+      `;
+    });
+    const [hero, coreSession] = await openBrowser(`/growing-list`);
+    const sessionId = await hero.sessionId;
+
+    for (let i = 0; i < 25; i += 1) {
+      await hero.getJsValue(`add('Text ${i}')`);
+      await collectElement(hero.document.querySelector('li:last-child'), 'item' + i);
+    }
+    for (let i = 0; i < 25; i += 1) {
+      const valid = await coreSession.getCollectedElements(sessionId, 'item' + i);
+      expect(valid).toHaveLength(1);
+      expect(valid[0].outerHTML).toBe(`<li>Text ${i}</li>`);
+    }
+  });
+});
+
+async function openBrowser(path: string): Promise<[Hero, CoreSession]> {
+  const hero = new Hero();
+  const coreSession = await hero[InternalPropertiesSymbol].coreSessionPromise;
+  Helpers.needsClosing.push(hero);
+  await hero.goto(`${koaServer.baseUrl}${path}`);
+  await hero.waitForPaintingStable();
+  return [hero, coreSession];
+}
+
+async function collectElement(
+  element: ISuperElement | ISuperNodeList,
+  name: string,
+): Promise<void> {
+  const { awaitedPath, awaitedOptions } = awaitedPathState.getState(element);
+  const coreFrame = await awaitedOptions.coreFrame;
+  await coreFrame.collectElement(name, awaitedPath.toJSON());
+}
diff --git a/fullstack/test/resources.test.ts b/fullstack/test/resources.test.ts
@@ -1,6 +1,7 @@
 import { Helpers } from '@ulixee/hero-testing';
 import { ITestKoaServer } from '@ulixee/hero-testing/helpers';
 import Hero from '../index';
+import { InternalPropertiesSymbol } from '@ulixee/hero/lib/InternalProperties';
 
 let koaServer: ITestKoaServer;
 beforeAll(async () => {
@@ -129,4 +130,42 @@ describe('basic resource tests', () => {
     await hero.close();
     await waitError;
   });
+
+  it('collects resources for extraction', async () => {
+    const hero1 = new Hero();
+    Helpers.needsClosing.push(hero1);
+    {
+      await hero1.goto(`${koaServer.baseUrl}/resources-test`);
+      await hero1.waitForPaintingStable();
+      const elem = hero1.document.querySelector('a');
+      await hero1.click(elem);
+
+      const resources = await hero1.waitForResource({ type: 'Fetch' });
+      expect(resources).toHaveLength(1);
+      const coreSession1 = await hero1[InternalPropertiesSymbol].coreSessionPromise;
+      const { resourceMeta, coreTabPromise } = resources[0][InternalPropertiesSymbol];
+      const coreTab = await coreTabPromise;
+      await coreTab.collectResource('xhr', resourceMeta.id);
+
+      const collected = await coreSession1.getCollectedResources(await hero1.sessionId, 'xhr');
+      expect(collected).toHaveLength(1);
+      expect(collected[0].response.json).toEqual({ hi: 'there' });
+      await hero1.close();
+    }
+
+    // Test that we can load a previous session too
+    {
+      const hero2 = new Hero();
+      Helpers.needsClosing.push(hero2);
+
+      await hero2.goto(`${koaServer.baseUrl}`);
+      await hero2.waitForPaintingStable();
+      const coreSession2 = await hero2[InternalPropertiesSymbol].coreSessionPromise;
+      const collected2 = await coreSession2.getCollectedResources(await hero1.sessionId, 'xhr');
+      expect(collected2).toHaveLength(1);
+      expect(collected2[0].url).toBe(`${koaServer.baseUrl}/ajax?counter=0`);
+      // should prefetch the body
+      expect(collected2[0].response.buffer).toBeTruthy();
+    }
+  });
 });
diff --git a/timetravel/lib/MirrorPage.ts b/timetravel/lib/MirrorPage.ts
@@ -111,11 +111,13 @@ export default class MirrorPage extends TypedEventEmitter<{
   }
 
   public async replaceDomRecording(domRecording: IDomRecording): Promise<void> {
-    this.setDomRecording(domRecording);
-    if (this.loadedDocument) {
-      this.isLoadedDocumentDirty = true;
-      await this.injectPaintEvents(this.loadedDocument);
-    }
+    await this.loadQueue.run(async () => {
+      this.setDomRecording(domRecording);
+      if (this.loadedDocument) {
+        this.isLoadedDocumentDirty = true;
+        await this.injectPaintEvents(this.loadedDocument);
+      }
+    });
   }
 
   public getPaintIndex(timestamp: number): number {
@@ -136,10 +138,14 @@ export default class MirrorPage extends TypedEventEmitter<{
     this.subscribeToTab = tab;
   }
 
-  public async load(newPaintIndex?: number, overlayLabel?: string): Promise<void> {
+  public async load<T = void>(
+    newPaintIndex?: number,
+    overlayLabel?: string,
+    afterLoadedCb?: () => Promise<T>,
+  ): Promise<T> {
     await this.isReady;
     // only allow 1 load at a time
-    await this.loadQueue.run(async () => {
+    return await this.loadQueue.run<T>(async () => {
       if (this.subscribeToTab && !newPaintIndex) {
         await this.subscribeToTab.flushDomChanges();
       }
@@ -196,6 +202,7 @@ export default class MirrorPage extends TypedEventEmitter<{
           await this.evaluate(`window.overlay(${JSON.stringify(options)});`);
         }
       }
+      if (afterLoadedCb) return await afterLoadedCb();
     });
   }
 
@@ -232,18 +239,23 @@ export default class MirrorPage extends TypedEventEmitter<{
     this.emit('close');
   }
 
-  public async getNodeOuterHtml(nodeId: number, frameDomNodeId?: number): Promise<string> {
-    await this.isReady;
-    const frame = await this.getFrameWithDomNodeId(frameDomNodeId);
-    return await frame.evaluate(
-      `(() => {
+  public async getNodeOuterHtml(
+    paintIndex: number,
+    nodeId: number,
+    frameDomNodeId?: number,
+  ): Promise<string> {
+    return await this.load<string>(paintIndex, null, async () => {
+      const frame = await this.getFrameWithDomNodeId(frameDomNodeId);
+      return await frame.evaluate(
+        `(() => {
      const node = NodeTracker.getWatchedNodeWithId(${nodeId});
      if (node) return node.outerHTML;
      return null;
    })()`,
-      true,
-      { retriesWaitingForLoad: 2 },
-    );
+        true,
+        { retriesWaitingForLoad: 2 },
+      );
+    });
   }
 
   public async getHtml(): Promise<string> {