diff --git a/src/api/test-controller/index.js b/src/api/test-controller/index.js index 82c815d2737..a6a8e74029f 100644 --- a/src/api/test-controller/index.js +++ b/src/api/test-controller/index.js @@ -57,6 +57,7 @@ import { SkipJsErrorsCommand, AddRequestHooksCommand, RemoveRequestHooksCommand, + ReportCommand, } from '../../test-run/commands/actions'; import { @@ -629,6 +630,10 @@ export default class TestController { return this.enqueueCommand(RemoveRequestHooksCommand, { hooks }); } + [delegatedAPI(ReportCommand.methodName)] (...args) { + return this.enqueueCommand(ReportCommand, { args }); + } + static enableDebugForNonDebugCommands () { inDebug = true; } diff --git a/src/reporter/index.ts b/src/reporter/index.ts index 3be3f774d10..83b79e03758 100644 --- a/src/reporter/index.ts +++ b/src/reporter/index.ts @@ -52,6 +52,7 @@ interface TaskInfo { } interface TestInfo { + reportData: Dictionary; fixture: Fixture; test: Test; testRunIds: string[]; @@ -86,6 +87,7 @@ interface BrowserRunInfo extends Browser { interface TestRunInfo { errs: TestRunErrorFormattableAdapter[]; warnings: string[]; + reportData: Dictionary; durationMs: number; unstable: boolean; screenshotPath: string; @@ -123,6 +125,11 @@ interface ReportWarningEventArguments { actionId?: string; } +interface ReportDataEventArgs { + data: any[]; + testRun: TestRun +} + const debugLog = debug('testcafe:reporter'); export default class Reporter { @@ -194,6 +201,8 @@ export default class Reporter { messageBus.on('warning-add', async e => await this._onWarningAddHandler(e)); + messageBus.on('report-data', async e => await this._onReportDataHandler(e)); + messageBus.once('start', async (task: Task) => await this._onceTaskStartHandler(task)); messageBus.on('test-run-start', async testRun => await this._onTaskTestRunStartHandler(testRun)); @@ -299,6 +308,7 @@ export default class Reporter { pendingTestRunDonePromise: Reporter._createPendingPromise(), pendingTestRunStartPromise: Reporter._createPendingPromise(), browsers: [], + reportData: {}, }; } @@ -312,6 +322,7 @@ export default class Reporter { return { errs: sortBy(reportItem.errs, ['userAgent', 'code']), warnings: reportItem.warnings, + reportData: reportItem.reportData, durationMs: +new Date() - (reportItem.startTime as number), //eslint-disable-line @typescript-eslint/no-extra-parens unstable: reportItem.unstable, screenshotPath: reportItem.screenshotPath as string, @@ -544,10 +555,12 @@ export default class Reporter { reportItem.browsers.push(browser); - reportItem.pendingRuns = isTestRunStoppedTaskExecution ? 0 : reportItem.pendingRuns - 1; - reportItem.unstable = reportItem.unstable || testRun.unstable; - reportItem.errs = reportItem.errs.concat(testRun.errs); - reportItem.warnings = testRun.warningLog ? union(reportItem.warnings, testRun.warningLog.messages) : []; + reportItem.pendingRuns = isTestRunStoppedTaskExecution ? 0 : reportItem.pendingRuns - 1; + reportItem.unstable = reportItem.unstable || testRun.unstable; + reportItem.errs = reportItem.errs.concat(testRun.errs); + reportItem.warnings = testRun.warningLog ? union(reportItem.warnings, testRun.warningLog.messages) : []; + reportItem.reportData = reportItem.reportData || {}; + reportItem.reportData[testRun.id] = testRun.reportDataLog ? testRun.reportDataLog.data : []; if (testRun.quarantine) { reportItem.quarantine = reportItem.quarantine || {}; @@ -635,4 +648,45 @@ export default class Reporter { (this.taskInfo.pendingTaskDonePromise.resolve as Function)(); } + + private _prepareReportDataEventArgs (testRun: TestRun): any { + const { test, browser, id } = testRun; + const fixture = test.fixture; + + const testInfo = { + name: test.name, + id: test.id, + meta: test.meta, + }; + + const fixtureInfo = { + name: fixture?.name, + id: fixture?.id, + meta: fixture?.meta, + path: fixture?.path, + }; + + return { + test: testInfo, + fixture: fixtureInfo, + testRunId: id, + browser, + }; + } + + private async _onReportDataHandler ({ testRun, data }: ReportDataEventArgs): Promise { + if (!this.taskInfo) + return; + + const testRunInfo = this._prepareReportDataEventArgs(testRun); + + await this.dispatchToPlugin({ + method: ReporterPluginMethod.reportData as string, + initialObject: this.taskInfo.task, + args: [ + testRunInfo, + ...data, + ], + }); + } } diff --git a/src/reporter/interfaces.ts b/src/reporter/interfaces.ts index b2576650f53..14857855c2e 100644 --- a/src/reporter/interfaces.ts +++ b/src/reporter/interfaces.ts @@ -10,6 +10,7 @@ export interface ReporterPlugin { reportTestDone(): void; reportTaskDone(): void; reportWarnings?(): void; + reportData? (): void } export interface ReporterSource { diff --git a/src/reporter/plugin-host.ts b/src/reporter/plugin-host.ts index 208899b11ee..353afb2c253 100644 --- a/src/reporter/plugin-host.ts +++ b/src/reporter/plugin-host.ts @@ -194,4 +194,8 @@ export default class ReporterPluginHost { // NOTE: It's an optional method public async reportWarnings (/* warnings */): Promise { // eslint-disable-line @typescript-eslint/no-empty-function } + + // NOTE: It's an optional method + public async reportData (/* testRun, ...data */): Promise { // eslint-disable-line @typescript-eslint/no-empty-function + } } diff --git a/src/reporter/plugin-methods.ts b/src/reporter/plugin-methods.ts index 5ba76b8da5f..46e0f103712 100644 --- a/src/reporter/plugin-methods.ts +++ b/src/reporter/plugin-methods.ts @@ -11,6 +11,7 @@ const ReporterPluginMethod: EnumFromPropertiesOf = { reportTestDone: 'reportTestDone', reportTaskDone: 'reportTaskDone', reportWarnings: 'reportWarnings', + reportData: 'reportData', }; export default ReporterPluginMethod; diff --git a/src/reporter/report-data-log.ts b/src/reporter/report-data-log.ts new file mode 100644 index 00000000000..8c317e3d3a8 --- /dev/null +++ b/src/reporter/report-data-log.ts @@ -0,0 +1,32 @@ +import MessageBus from '../utils/message-bus'; +import TestRun from '../test-run'; + +type ReportDataLogCallback = (data: any[]) => Promise; + +export default class ReportDataLog { + private readonly _data: any[]; + public callback?: ReportDataLogCallback; + + public constructor (callback?: ReportDataLogCallback) { + this._data = []; + this.callback = callback; + } + + public get data (): any[] { + return this._data; + } + + public async addData (data: any[]): Promise { + if (this.callback) + await this.callback(data); + + this._data.push(...data); + } + + public static createAddDataCallback (messageBus: MessageBus | undefined, testRun: TestRun): ReportDataLogCallback { + return async (data: any[]) => { + if (messageBus) + await messageBus.emit('report-data', { data, testRun }); + }; + } +} diff --git a/src/services/serialization/replicator/transforms/command-base-trasform/command-constructors.ts b/src/services/serialization/replicator/transforms/command-base-trasform/command-constructors.ts index 9f98ec8c081..f437a8e5ca4 100644 --- a/src/services/serialization/replicator/transforms/command-base-trasform/command-constructors.ts +++ b/src/services/serialization/replicator/transforms/command-base-trasform/command-constructors.ts @@ -49,6 +49,7 @@ import { SkipJsErrorsCommand, AddRequestHooksCommand, RemoveRequestHooksCommand, + ReportCommand, } from '../../../../../test-run/commands/actions'; import { AssertionCommand } from '../../../../../test-run/commands/assertion'; @@ -115,6 +116,7 @@ const COMMAND_CONSTRUCTORS = new Map([ [CommandType.skipJsErrors, SkipJsErrorsCommand], [CommandType.addRequestHooks, AddRequestHooksCommand], [CommandType.removeRequestHooks, RemoveRequestHooksCommand], + [CommandType.report, ReportCommand], ]); export default COMMAND_CONSTRUCTORS; diff --git a/src/test-run/commands/actions.d.ts b/src/test-run/commands/actions.d.ts index 513f3a2fbcc..b9dd7c8ee85 100644 --- a/src/test-run/commands/actions.d.ts +++ b/src/test-run/commands/actions.d.ts @@ -285,3 +285,8 @@ export class RunCustomActionCommand extends ActionCommandBase { public args: any; } +export class ReportCommand extends ActionCommandBase { + public constructor (obj: object, testRun: TestRun, validateProperties: boolean); + public args: any[]; +} + diff --git a/src/test-run/commands/actions.js b/src/test-run/commands/actions.js index b0820eec638..60b67d86e55 100644 --- a/src/test-run/commands/actions.js +++ b/src/test-run/commands/actions.js @@ -813,3 +813,16 @@ export class RemoveRequestHooksCommand extends ActionCommandBase { } } +export class ReportCommand extends ActionCommandBase { + static methodName = camelCase(TYPE.report); + + constructor (obj, testRun, validateProperties) { + super(obj, testRun, TYPE.report, validateProperties); + } + + getAssignableProperties () { + return [ + { name: 'args', required: true }, + ]; + } +} diff --git a/src/test-run/commands/type.js b/src/test-run/commands/type.js index 8a63204aef2..8b9cdc13813 100644 --- a/src/test-run/commands/type.js +++ b/src/test-run/commands/type.js @@ -72,4 +72,5 @@ export default { addRequestHooks: 'add-request-hooks', removeRequestHooks: 'remove-request-hooks', runCustomAction: 'run-custom-action', + report: 'report', }; diff --git a/src/test-run/index.ts b/src/test-run/index.ts index 5d267501541..de96ca8cfb2 100644 --- a/src/test-run/index.ts +++ b/src/test-run/index.ts @@ -143,6 +143,7 @@ import { import ProxylessRequestPipeline from '../proxyless/request-pipeline'; import Proxyless from '../proxyless'; +import ReportDataLog from '../reporter/report-data-log'; const lazyRequire = require('import-lazy')(require); const ClientFunctionBuilder = lazyRequire('../client-functions/client-function-builder'); @@ -227,6 +228,7 @@ interface OpenedWindowInformation { export default class TestRun extends AsyncEventEmitter { private [testRunMarker]: boolean; public readonly warningLog: WarningLog; + public readonly reportDataLog: ReportDataLog; private readonly opts: Dictionary; public readonly test: Test; public readonly browserConnection: BrowserConnection; @@ -290,6 +292,7 @@ export default class TestRun extends AsyncEventEmitter { this[testRunMarker] = true; this._messageBus = messageBus; this.warningLog = new WarningLog(globalWarningLog, WarningLog.createAddWarningCallback(messageBus, this)); + this.reportDataLog = new ReportDataLog(ReportDataLog.createAddDataCallback(messageBus, this)); this.opts = opts; this.test = test; this.browserConnection = browserConnection; @@ -307,8 +310,8 @@ export default class TestRun extends AsyncEventEmitter { this.pageLoadTimeout = this._getPageLoadTimeout(test, opts); this.testExecutionTimeout = this._getTestExecutionTimeout(opts); - this.disablePageReloads = test.disablePageReloads || opts.disablePageReloads as boolean && test.disablePageReloads !== false; - this.disablePageCaching = test.disablePageCaching || opts.disablePageCaching as boolean; + this.disablePageReloads = test.disablePageReloads || opts.disablePageReloads as boolean && test.disablePageReloads !== false; + this.disablePageCaching = test.disablePageCaching || opts.disablePageCaching as boolean; this.disableMultipleWindows = opts.disableMultipleWindows as boolean; @@ -350,7 +353,7 @@ export default class TestRun extends AsyncEventEmitter { this.debugLog = new TestRunDebugLog(this.browserConnection.userAgent); - this.quarantine = null; + this.quarantine = null; this.debugLogger = this.opts.debugLogger; @@ -1308,6 +1311,9 @@ export default class TestRun extends AsyncEventEmitter { return await wrappedFn(this, args); } + if (command.type === COMMAND_TYPE.report) + return await this.reportDataLog.addData(command.args as any[]); + if (command.type === COMMAND_TYPE.assertion) return this._executeAssertion(command as AssertionCommand, callsite as CallsiteRecord); diff --git a/test/functional/fixtures/reporter/test.js b/test/functional/fixtures/reporter/test.js index 0acd462229e..517f505f4e9 100644 --- a/test/functional/fixtures/reporter/test.js +++ b/test/functional/fixtures/reporter/test.js @@ -888,6 +888,61 @@ const experimentalDebug = !!process.env.EXPERIMENTAL_DEBUG; }); }); + describe('Report Data', () => { + let reportDataInfos = {}; + let testDoneInfos = {}; + let reporter = null; + + const createReportDataReporter = () => { + return createReporter({ + reportData: ({ browser }, ...data) => { + const alias = browser.alias; + + if (!reportDataInfos[alias]) + reportDataInfos[alias] = []; + + reportDataInfos[alias].push(data); + }, + reportTestDone: (name, { reportData, browsers }) => { + browsers.forEach(({ testRunId, alias }) => { + testDoneInfos[alias] = reportData[testRunId]; + }); + }, + }); + }; + + beforeEach(() => { + reportDataInfos = {}; + testDoneInfos = {}; + + reporter = createReportDataReporter(); + }); + + it('Should raise "reportData" event', async () => { + const expectedReportData = [1, true, 'string', { 'reportResult': 'test' }]; + + await runTests('testcafe-fixtures/report-data-test.js', 'Run t.report action', { + reporter, + }); + + const reportDataBrowserInfos = Object.entries(reportDataInfos); + const testDoneBrowserInfos = Object.entries(testDoneInfos); + + expect(reportDataBrowserInfos.length).eql(config.browsers.length); + expect(testDoneBrowserInfos.length).eql(config.browsers.length); + + reportDataBrowserInfos.forEach(([alias, reportData]) => { + expect(reportData.flat()).eql(testDoneInfos[alias]); + }); + + testDoneBrowserInfos.forEach(([, reportData]) => { + const [, ...rest] = reportData; + + expect(rest).eql(expectedReportData); + }); + }); + }); + describe('Warnings', () => { let warningResult = {}; let reporter = null; diff --git a/test/functional/fixtures/reporter/testcafe-fixtures/index-test.js b/test/functional/fixtures/reporter/testcafe-fixtures/index-test.js index 4096e47b8b0..3e30a8a3e27 100644 --- a/test/functional/fixtures/reporter/testcafe-fixtures/index-test.js +++ b/test/functional/fixtures/reporter/testcafe-fixtures/index-test.js @@ -25,6 +25,7 @@ async function errorCheck (t) { test('Simple test', async t => { await t.wait(1); + await t.report(); }); test('Simple command test', async t => { diff --git a/test/functional/fixtures/reporter/testcafe-fixtures/report-data-test.js b/test/functional/fixtures/reporter/testcafe-fixtures/report-data-test.js new file mode 100644 index 00000000000..8bbded0e186 --- /dev/null +++ b/test/functional/fixtures/reporter/testcafe-fixtures/report-data-test.js @@ -0,0 +1,8 @@ +fixture`Report Data API` + .page('../pages/index.html'); + +test('Run t.report action', async t => { + await t + .report(t.browser.alias) + .report(1, true, 'string', { 'reportResult': 'test' }); +}); diff --git a/test/server/data/test-controller-reporter-expected/index.js b/test/server/data/test-controller-reporter-expected/index.js index e7572519be4..4c4710fd8a1 100644 --- a/test/server/data/test-controller-reporter-expected/index.js +++ b/test/server/data/test-controller-reporter-expected/index.js @@ -877,16 +877,28 @@ module.exports = { browser: { alias: 'test-browser', headless: false }, }, { - testRunId: "test-run-id", - name: "skipJsErrors", + testRunId: 'test-run-id', + name: 'skipJsErrors', command: { - actionId: "SkipJsErrorsCommand", + actionId: 'SkipJsErrorsCommand', options: true, - type: "skip-js-errors", + type: 'skip-js-errors', + }, + test: { id: 'test-id', name: 'test-name', phase: 'initial' }, + fixture: { id: 'fixture-id', name: 'fixture-name' }, + browser: { alias: 'test-browser', headless: false }, + }, + { + testRunId: 'test-run-id', + name: 'report', + command: { + actionId: 'ReportCommand', + args: [], + type: 'report', }, - test: { id: "test-id", name: "test-name", phase: "initial" }, - fixture: { id: "fixture-id", name: "fixture-name" }, - browser: { alias: "test-browser", headless: false }, + test: { id: 'test-id', name: 'test-name', phase: 'initial' }, + fixture: { id: 'fixture-id', name: 'fixture-name' }, + browser: { alias: 'test-browser', headless: false }, }, ], } diff --git a/test/server/reporter-test.js b/test/server/reporter-test.js index 4210a667437..05c5fc785b5 100644 --- a/test/server/reporter-test.js +++ b/test/server/reporter-test.js @@ -189,6 +189,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, quarantine: { attempts: [ { testRunId: 'firstRunId', errors: ['1', '2'] }, @@ -205,6 +206,7 @@ describe('Reporter', () => { unstable: false, browserConnection: browserConnectionMocks[0], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], errs: [ @@ -221,6 +223,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, @@ -232,6 +235,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, @@ -243,6 +247,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, @@ -254,6 +259,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, @@ -265,6 +271,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, @@ -276,6 +283,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[0], errs: [], warningLog: { messages: ['warning2'] }, + reportDataLog: { data: [] }, browser: browserMocks[0], }, ]; @@ -289,6 +297,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, quarantine: { attempts: [ { testRunId: 'firstRunId', errors: ['1', '2'] }, @@ -306,6 +315,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [{ text: 'err1' }], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -317,6 +327,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -328,6 +339,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -339,6 +351,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -350,6 +363,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [{ text: 'err1' }], warningLog: { messages: ['warning1'] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -361,6 +375,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: [] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, @@ -372,6 +387,7 @@ describe('Reporter', () => { browserConnection: browserConnectionMocks[1], errs: [], warningLog: { messages: ['warning2', 'warning3'] }, + reportDataLog: { data: [] }, browser: browserMocks[1], }, ]; @@ -498,7 +514,7 @@ describe('Reporter', () => { createReporter(messageBus); - expect(messageBus.listenerCount()).eql(8); + expect(messageBus.listenerCount()).eql(9); expect(messageBus.listenerCount('warning-add')).eql(1); expect(messageBus.listenerCount('start')).eql(1); expect(messageBus.listenerCount('test-run-start')).eql(1); @@ -507,6 +523,7 @@ describe('Reporter', () => { expect(messageBus.listenerCount('test-action-done')).eql(1); expect(messageBus.listenerCount('done')).eql(1); expect(messageBus.listenerCount('unhandled-rejection')).eql(1); + expect(messageBus.listenerCount('report-data')).eql(1); }); it('Should analyze task progress and call appropriate plugin methods', function () { @@ -767,6 +784,10 @@ describe('Reporter', () => { run: 'run-001', }, }, + reportData: { + 'f1t1': [], + 'f1t1ff': [], + }, }, { run: 'run-001', @@ -860,6 +881,10 @@ describe('Reporter', () => { run: 'run-001', }, }, + reportData: { + f1t2: [], + f1t2ff: [], + }, }, { run: 'run-001', @@ -925,6 +950,10 @@ describe('Reporter', () => { run: 'run-001', }, }, + reportData: { + f1t3: [], + f1t3ff: [], + }, }, { run: 'run-001', @@ -1000,6 +1029,10 @@ describe('Reporter', () => { run: 'run-002', }, }, + reportData: { + f2t1: [], + f2t1ff: [], + }, }, { run: 'run-001', @@ -1065,6 +1098,10 @@ describe('Reporter', () => { run: 'run-002', }, }, + reportData: { + f2t2: [], + f2t2ff: [], + }, }, { run: 'run-001', @@ -1142,6 +1179,10 @@ describe('Reporter', () => { path: './file2.js', meta: null, }, + reportData: { + f3t1: [], + f3t1ff: [], + }, }, { run: 'run-001', @@ -1205,6 +1246,10 @@ describe('Reporter', () => { path: './file2.js', meta: null, }, + reportData: { + f3t2: [], + f3t2ff: [], + }, }, { run: 'run-001', @@ -1268,6 +1313,10 @@ describe('Reporter', () => { path: './file2.js', meta: null, }, + reportData: { + f3t3: [], + f3t3ff: [], + }, }, { run: 'run-001', @@ -1371,7 +1420,7 @@ describe('Reporter', () => { [ { testRunId: 'f1t2-id1', videoPath: 'f1t2-path1' }, { testRunId: 'f1t2-id2', videoPath: 'f1t2-path2' }, - ]] + ]], ); }); }); diff --git a/test/server/test-controller-events-test.js b/test/server/test-controller-events-test.js index 247ba133173..14180873e37 100644 --- a/test/server/test-controller-events-test.js +++ b/test/server/test-controller-events-test.js @@ -141,6 +141,7 @@ const actions = { setCookies: [{ cookieName: 'cookieValue' }, 'https://domain.com'], deleteCookies: [['cookieName1', 'cookieName2'], 'https://domain.com'], skipJsErrors: [true], + report: [], }; let testController = null; diff --git a/ts-defs-src/test-api/test-controller.d.ts b/ts-defs-src/test-api/test-controller.d.ts index 8532492c7ac..34ab7300699 100644 --- a/ts-defs-src/test-api/test-controller.d.ts +++ b/ts-defs-src/test-api/test-controller.d.ts @@ -548,6 +548,13 @@ interface TestController { * @param options - Error skipping conditions: a Boolean flag, an Object with options, or a callback function that defines custom error skipping logic. */ skipJsErrors (options?: boolean | SkipJsErrorsOptionsObject | SkipJsErrorsCallback | SkipJsErrorsCallbackWithOptionsObject): TestControllerPromise; + + /** + * Pass additional data to the reporter. + * + * @param args - Any user data. + */ + report (...args: any[]): TestControllerPromise; } interface TestControllerPromise extends TestController, Promise {