diff --git a/.github/workflows/js-branch.yml b/.github/workflows/js-branch.yml new file mode 100644 index 000000000..f12d1bb10 --- /dev/null +++ b/.github/workflows/js-branch.yml @@ -0,0 +1,50 @@ +name: 'Publish a built Javascript Branch' + +on: + push: + branches: + workflow_dispatch: + +jobs: + build: + name: Build Javascript + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: 18 + cache: yarn + + - name: Clone ulixee/shared + run: git clone https://github.com/ulixee/shared.git + working-directory: ../.. + + - name: Install ulixee/shared + run: yarn build + working-directory: ../../shared + + - name: Clone unblocked + run: git clone --recurse-submodules -j8 https://github.com/ulixee/unblocked.git + working-directory: ../.. + + - name: Install unblocked + run: yarn build + working-directory: ../../unblocked + + - name: Build modules + run: yarn && yarn build:dist --network-timeout 1000000 + + - name: Publish branch + run: | + cd build-dist + git config --global user.email "staff@ulixee.org" + git config --global user.name "CI" + git init -b main + git add -A + git commit -m 'Auto-build Javascript files' + git push -f https://ulixee:${{ env.GH_TOKEN }}@github.com/ulixee/hero.git main:${{ github.ref_name }}-built-js + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/core/dbs/SessionDb.ts b/core/dbs/SessionDb.ts index a1ade2fab..9866fd672 100644 --- a/core/dbs/SessionDb.ts +++ b/core/dbs/SessionDb.ts @@ -191,8 +191,8 @@ export default class SessionDb { } SessionDb.byId.delete(this.sessionId); + this.db.close(); if (deleteFile) { - this.db.close(); await Fs.promises.rm(this.path); } this.db = null; diff --git a/core/lib/CommandRecorder.ts b/core/lib/CommandRecorder.ts index 835fea70e..758eca775 100644 --- a/core/lib/CommandRecorder.ts +++ b/core/lib/CommandRecorder.ts @@ -9,6 +9,7 @@ type AsyncFunc = (...args: any[]) => Promise; export default class CommandRecorder { public readonly fnNames = new Set(); + private readonly fnMap = new Map(); private logger: IBoundLog; private isClosed = false; @@ -20,8 +21,10 @@ export default class CommandRecorder { fns: AsyncFunc[], ) { for (const fn of fns) { + // used for bypassing recording owner[`___${fn.name}`] = fn.bind(owner); - owner[fn.name] = ((...args) => this.runCommandFn(fn, ...args)) as any; + this.fnMap.set(fn.name, fn.bind(owner)); + owner[fn.name] = this.runCommandFn.bind(this, fn.name); this.fnNames.add(fn.name); } this.logger = log.createChild(module, { @@ -35,12 +38,14 @@ export default class CommandRecorder { this.isClosed = true; this.session = null; this.owner = null; + this.fnMap.clear(); } - private async runCommandFn(commandFn: AsyncFunc, ...args: any[]): Promise { + private async runCommandFn(functionName: string, ...args: any[]): Promise { if (this.isClosed) return; - if (!this.fnNames.has(commandFn.name)) - throw new Error(`Unsupported function requested ${commandFn.name}`); + const commandFn = this.fnMap.get(functionName); + if (!this.fnNames.has(functionName) || !commandFn) + throw new Error(`Unsupported function requested ${functionName}`); const { session, owner } = this; if (session === null) return; @@ -50,7 +55,7 @@ export default class CommandRecorder { session.commands.presetMeta = null; const shouldWait = - !owner.shouldWaitForCommandLock || owner.shouldWaitForCommandLock(commandFn.name); + !owner.shouldWaitForCommandLock || owner.shouldWaitForCommandLock(functionName); if (shouldWait) await commands.waitForCommandLock(); let tabId = this.tabId; @@ -68,7 +73,7 @@ export default class CommandRecorder { tabId, frameId, frame?.navigations?.top?.id, - commandFn.name, + functionName, args, meta, ); diff --git a/core/lib/Session.ts b/core/lib/Session.ts index 92a332450..161384075 100644 --- a/core/lib/Session.ts +++ b/core/lib/Session.ts @@ -68,7 +68,7 @@ export default class Session public readonly id: string; public readonly baseDir: string; - public readonly plugins: CorePlugins; + public plugins: CorePlugins; public get browserEngine(): IBrowserEngine { return this.emulationProfile.browserEngine; @@ -171,7 +171,7 @@ export default class Session 'rerun-kept-alive': void; }>(); - public readonly agent: Agent; + public agent: Agent; protected readonly logger: IBoundLog; @@ -195,7 +195,6 @@ export default class Session this.id = this.getId(options.sessionId); const id = this.id; Session.byId[id] = this; - this.events.once(this, 'closed', () => delete Session.byId[id]); this.db = new SessionDb(this.id); this.commands = new Commands(this.db); @@ -469,6 +468,7 @@ export default class Session } public getLastActiveTab(): Tab { + if (!this.commands) return null; for (let idx = this.commands.history.length - 1; idx >= 0; idx -= 1) { const command = this.commands.history[idx]; if (command.tabId) { @@ -562,6 +562,7 @@ export default class Session this.emit('closed', closedEvent); await closedEvent.waitForPromise; + delete Session.byId[this.id]; this.events.close(); this.commandRecorder.cleanup(); this.plugins.cleanup(); @@ -571,6 +572,7 @@ export default class Session LogEvents.unsubscribe(this.logSubscriptionId); loggerSessionIdNames.delete(this.id); this.db.flush(); + this.cleanup(); this.removeAllListeners(); try { @@ -660,6 +662,14 @@ export default class Session return sessionId ?? nanoid(); } + private cleanup(): void { + this.agent = null; + this.commandRecorder = null; + this.browserContext = null; + this.plugins = null; + this.commands = null; + } + private onResource(event: BrowserContext['resources']['EventTypes']['change']): void { this.db.resources.insert( event.tabId, diff --git a/core/lib/Tab.ts b/core/lib/Tab.ts index e8b3c981f..3f3f712bf 100644 --- a/core/lib/Tab.ts +++ b/core/lib/Tab.ts @@ -67,7 +67,7 @@ export default class Tab } public readonly parentTabId?: number; - public readonly session: Session; + public session: Session; public readonly frameEnvironmentsById = new Map(); public readonly frameEnvironmentsByDevtoolsId = new Map(); public page: Page; @@ -247,9 +247,7 @@ export default class Tab await this.page.setJavaScriptEnabled(enableJs); } - public setBlockedResourceUrls( - blockedUrls: (string | RegExp)[], - ): void { + public setBlockedResourceUrls(blockedUrls: (string | RegExp)[]): void { const mitmSession = this.session.mitmRequestSession; let interceptor = mitmSession.interceptorHandlers.find(x => x.types && !x.handlerFn); @@ -308,10 +306,14 @@ export default class Tab } } this.events.close(); + this.commandRecorder.cleanup(); this.commandRecorder = null; this.emit('close'); // clean up listener memory this.removeAllListeners(); + this.session = null; + this.frameEnvironmentsById.clear(); + this.frameEnvironmentsByDevtoolsId.clear(); this.logger.stats('Tab.Closed', { parentLogId, errors }); } @@ -841,14 +843,10 @@ export default class Tab private async waitForReady(): Promise { await this.mainFrameEnvironment.isReady; if (this.session.options?.blockedResourceTypes) { - await this.setBlockedResourceTypes( - this.session.options.blockedResourceTypes, - ); + await this.setBlockedResourceTypes(this.session.options.blockedResourceTypes); } if (this.session.options?.blockedResourceUrls) { - await this.setBlockedResourceUrls( - this.session.options.blockedResourceUrls, - ); + await this.setBlockedResourceUrls(this.session.options.blockedResourceUrls); } } diff --git a/core/test/user-profile.test.ts b/core/test/user-profile.test.ts index ffd7e5812..9007cdadb 100644 --- a/core/test/user-profile.test.ts +++ b/core/test/user-profile.test.ts @@ -523,9 +523,9 @@ document.querySelector('#local').innerHTML = localStorage.getItem('local'); 'textContent', ]); expect(crossContent.value).toBe('1'); + const history = tab.navigations.history; await session.close(); - const history = tab.navigations.history; expect(history).toHaveLength(1); expect(history[0].finalUrl).toBe(`${koaServer.baseUrl}/cross-storage2`); } diff --git a/timetravel/lib/DomStateGenerator.ts b/timetravel/lib/DomStateGenerator.ts index 2efb354c5..f4b3192da 100644 --- a/timetravel/lib/DomStateGenerator.ts +++ b/timetravel/lib/DomStateGenerator.ts @@ -13,6 +13,7 @@ import IResourceFilterProperties from '@ulixee/hero-interfaces/IResourceFilterPr import { IStorageChangesEntry } from '@ulixee/hero-core/models/StorageChangesTable'; import Resolvable from '@ulixee/commons/lib/Resolvable'; import BrowserContext from '@ulixee/unblocked-agent/lib/BrowserContext'; +import Session from '@ulixee/hero-core/lib/Session'; import { NodeType } from './DomNode'; import DomRebuilder from './DomRebuilder'; import MirrorPage from './MirrorPage'; @@ -48,7 +49,10 @@ export default class DomStateGenerator { sessionId, needsProcessing: true, mainFrameIds: sessionDb.frames.mainFrameIds(), - db: sessionDb, + // could get closed, so need to use getter + get db(): SessionDb { + return SessionDb.getCached(sessionId, false); + }, dbLocation: SessionDb.databaseDir, loadingRange: [...loadingRange], timelineRange: timelineRange ? [...timelineRange] : undefined, @@ -83,15 +87,18 @@ export default class DomStateGenerator { }; } for (const session of savedState.sessions) { + const sessionId = session.sessionId; let db: SessionDb; try { - db = SessionDb.getCached(session.sessionId, false); + db = SessionDb.getCached(sessionId, false); } catch (err) { // couldn't load } - this.sessionsById.set(session.sessionId, { + this.sessionsById.set(sessionId, { ...session, - db, + get db(): SessionDb { + return SessionDb.getCached(sessionId, false); + }, needsProcessing: !!db, mainFrameIds: db?.frames.mainFrameIds(session.tabId), }); @@ -188,7 +195,11 @@ export default class DomStateGenerator { // wait for end point if (timeoutMs > 0) await new Promise(resolve => setTimeout(resolve, timeoutMs)); - if (!db.readonly) db.flush(); + + const liveSession = Session.get(sessionId); + if (liveSession && liveSession.db?.isOpen && !liveSession.db?.readonly) { + liveSession.db.flush(); + } session.mainFrameIds = db.frames.mainFrameIds(tabId); diff --git a/timetravel/lib/MirrorPage.ts b/timetravel/lib/MirrorPage.ts index 3d230427c..a447a7b8b 100644 --- a/timetravel/lib/MirrorPage.ts +++ b/timetravel/lib/MirrorPage.ts @@ -64,10 +64,12 @@ export default class MirrorPage extends TypedEventEmitter<{ super(); this.setDomRecording(domRecording); this.onPageEvents = this.onPageEvents.bind(this); + this.logger = log.createChild(module); } public async attachToPage(page: Page, sessionId: string, setReady = true): Promise { this.page = page; + this.logger = log.createChild(module, { sessionId }); let readyResolvable: Resolvable; if (setReady) { readyResolvable = new Resolvable(); @@ -77,16 +79,10 @@ export default class MirrorPage extends TypedEventEmitter<{ this.events.once(page, 'close', this.close.bind(this)); if (this.debugLogging) { this.events.on(page, 'console', msg => { - log.info('MirrorPage.console', { - ...msg, - sessionId, - }); + this.logger.info('MirrorPage.console', msg); }); this.events.on(page, 'crashed', msg => { - log.info('MirrorPage.crashed', { - ...msg, - sessionId, - }); + this.logger.info('MirrorPage.crashed', msg); }); } @@ -123,6 +119,7 @@ export default class MirrorPage extends TypedEventEmitter<{ if (this.isReady) return await this.isReady; this.sessionId = sessionId; + this.logger = log.createChild(module, { sessionId }); const ready = new Resolvable(); this.isReady = ready.promise; try { @@ -209,10 +206,9 @@ export default class MirrorPage extends TypedEventEmitter<{ isLoadingDocument = true; if (this.debugLogging) { - log.info('MirrorPage.navigate', { + this.logger.info('MirrorPage.navigate', { newPaintIndex, url: loadingDocument.url, - sessionId: this.sessionId, }); } const loader = await this.page.navigate(loadingDocument.url); @@ -229,9 +225,8 @@ export default class MirrorPage extends TypedEventEmitter<{ if (newPaintIndex >= 0) { if (this.debugLogging) { - log.info('MirrorPage.loadPaintEvents', { + this.logger.info('MirrorPage.loadPaintEvents', { newPaintIndex, - sessionId: this.sessionId, }); } @@ -272,8 +267,6 @@ export default class MirrorPage extends TypedEventEmitter<{ } public async close(): Promise { - if (this.isReady === null) return; - this.isReady = null; this.loadQueue.stop(); if (this.page && !this.page.isClosed) { await this.page.close(); @@ -337,9 +330,8 @@ export default class MirrorPage extends TypedEventEmitter<{ return frame; } } catch (error) { - log.warn('Error matching frame nodeId to environment', { + this.logger.warn('Error matching frame nodeId to environment', { error, - sessionId: this.sessionId, }); // just keep looking? }