From 2939b88dd25ddd68e9483aebc484fd67f7b14c2b Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Tue, 21 May 2024 12:30:17 +0200 Subject: [PATCH 1/5] support parallel mode for mocha --- ci/init.js | 7 + ext/exporters.d.ts | 1 + ext/exporters.js | 3 +- integration-tests/ci-visibility.spec.js | 136 +++- .../ci-visibility/run-workerpool.js | 23 + .../src/helpers/hooks.js | 1 + .../datadog-instrumentations/src/mocha.js | 677 +---------------- .../src/mocha/common.js | 50 ++ .../src/mocha/main.js | 678 ++++++++++++++++++ .../src/mocha/utils.js | 75 ++ .../src/mocha/worker.js | 199 +++++ packages/datadog-plugin-mocha/src/index.js | 90 ++- .../exporters/test-worker/index.js | 6 +- packages/dd-trace/src/exporter.js | 1 + packages/dd-trace/src/plugins/index.js | 1 + packages/dd-trace/src/plugins/util/test.js | 6 + .../exporters/test-worker/exporter.spec.js | 30 +- 17 files changed, 1292 insertions(+), 692 deletions(-) create mode 100644 integration-tests/ci-visibility/run-workerpool.js create mode 100644 packages/datadog-instrumentations/src/mocha/common.js create mode 100644 packages/datadog-instrumentations/src/mocha/main.js create mode 100644 packages/datadog-instrumentations/src/mocha/utils.js create mode 100644 packages/datadog-instrumentations/src/mocha/worker.js diff --git a/ci/init.js b/ci/init.js index 3599b2e05f4..b54e29abd4d 100644 --- a/ci/init.js +++ b/ci/init.js @@ -4,6 +4,7 @@ const { isTrue } = require('../packages/dd-trace/src/util') const isJestWorker = !!process.env.JEST_WORKER_ID const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID +const isMochaWorker = !!process.env.MOCHA_WORKER_ID const options = { startupLogs: false, @@ -44,6 +45,12 @@ if (isCucumberWorker) { } } +if (isMochaWorker) { + options.experimental = { + exporter: 'mocha_worker' + } +} + if (shouldInit) { tracer.init(options) tracer.use('fs', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index d2ebaefe267..07bc2cd29e3 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -5,6 +5,7 @@ declare const exporters: { AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', CUCUMBER_WORKER: 'cucumber_worker' + MOCHA_WORKER: 'mocha_worker' } export = exporters diff --git a/ext/exporters.js b/ext/exporters.js index b615d28f459..770116c3152 100644 --- a/ext/exporters.js +++ b/ext/exporters.js @@ -5,5 +5,6 @@ module.exports = { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', + MOCHA_WORKER: 'mocha_worker' } diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js index e4c2eb65e70..34e5eda8468 100644 --- a/integration-tests/ci-visibility.spec.js +++ b/integration-tests/ci-visibility.spec.js @@ -31,7 +31,10 @@ const { TEST_EARLY_FLAKE_ENABLED, TEST_NAME, JEST_DISPLAY_NAME, - TEST_EARLY_FLAKE_ABORT_REASON + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_COMMAND, + TEST_MODULE, + MOCHA_IS_PARALLEL } = require('../packages/dd-trace/src/plugins/util/test') const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') @@ -58,7 +61,7 @@ const testFrameworks = [ { ...mochaCommonOptions, testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each'], + dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], expectedCoverageFiles: [ 'ci-visibility/run-mocha.js', 'ci-visibility/test/sum.js', @@ -152,11 +155,47 @@ testFrameworks.forEach(({ }) }).timeout(50000) - it('does not init CI Visibility when running in parallel mode', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) + it('works with parallel mode', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal( + sessionEventContent.test_session_id.toString(10), + moduleEventContent.test_session_id.toString(10) + ) + suites.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + }) + + tests.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + }) + }) childProcess = fork(testFile, { cwd, @@ -175,7 +214,65 @@ testFrameworks.forEach(({ testOutput += chunk.toString() }) childProcess.on('message', () => { - assert.include(testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.') + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('works with parallel mode when run with the cli', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal(suites.length, 2) + assert.equal(tests.length, 2) + }) + childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('does not blow up when workerpool is used outside of a test', (done) => { + childProcess = exec('node ./ci-visibility/run-workerpool.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', (code) => { + assert.include(testOutput, 'result 7') + assert.equal(code, 0) done() }) }) @@ -1643,6 +1740,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('does not init if DD_API_KEY is not set', (done) => { receiver.assertMessageReceived(() => { done(new Error('Should not create spans')) @@ -1671,6 +1769,7 @@ testFrameworks.forEach(({ done() }) }) + it('can report git metadata', (done) => { const searchCommitsRequestPromise = receiver.payloadReceived( ({ url }) => url === '/api/v2/git/repository/search_commits' @@ -1702,6 +1801,7 @@ testFrameworks.forEach(({ stdio: 'pipe' }) }) + it('can report code coverage', (done) => { let testOutput const libraryConfigRequestPromise = receiver.payloadReceived( @@ -1765,6 +1865,7 @@ testFrameworks.forEach(({ done() }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1801,6 +1902,7 @@ testFrameworks.forEach(({ } ) }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -1862,6 +1964,7 @@ testFrameworks.forEach(({ } ) }) + it('marks the test session as skipped if every suite is skipped', (done) => { receiver.setSuitesToSkip( [ @@ -1900,6 +2003,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('does not skip tests if git metadata upload fails', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -1943,6 +2047,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.setSettings({ itr_enabled: true, @@ -1982,6 +2087,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip suites if suite is marked as unskippable', (done) => { receiver.setSuitesToSkip([ { @@ -2062,6 +2168,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { receiver.setSuitesToSkip([ { @@ -2136,6 +2243,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2170,6 +2278,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in test suites', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) @@ -2238,6 +2347,7 @@ testFrameworks.forEach(({ }) }) }) + it('reports errors in test sessions', (done) => { const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -2272,6 +2382,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('can report git metadata', (done) => { const infoRequestPromise = receiver.payloadReceived(({ url }) => url === '/info') const searchCommitsRequestPromise = receiver.payloadReceived( @@ -2311,6 +2422,7 @@ testFrameworks.forEach(({ stdio: 'pipe' }) }) + it('can report code coverage', (done) => { let testOutput const libraryConfigRequestPromise = receiver.payloadReceived( @@ -2375,6 +2487,7 @@ testFrameworks.forEach(({ done() }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ itr_enabled: false, @@ -2405,6 +2518,7 @@ testFrameworks.forEach(({ } ) }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2460,6 +2574,7 @@ testFrameworks.forEach(({ } ) }) + it('marks the test session as skipped if every suite is skipped', (done) => { receiver.setSuitesToSkip( [ @@ -2498,6 +2613,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('marks the test session as skipped if every suite is skipped', (done) => { receiver.setSuitesToSkip( [ @@ -2536,6 +2652,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('does not skip tests if git metadata upload fails', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not request skippable') @@ -2572,6 +2689,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not request skippable') @@ -2612,6 +2730,7 @@ testFrameworks.forEach(({ } ) }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2646,6 +2765,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in test suites', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) diff --git a/integration-tests/ci-visibility/run-workerpool.js b/integration-tests/ci-visibility/run-workerpool.js new file mode 100644 index 00000000000..8a77c9e315b --- /dev/null +++ b/integration-tests/ci-visibility/run-workerpool.js @@ -0,0 +1,23 @@ +// eslint-disable-next-line +const workerpool = require('workerpool') +const pool = workerpool.pool({ workerType: 'process' }) + +function add (a, b) { + return a + b +} + +pool + .exec(add, [3, 4]) + .then((result) => { + // eslint-disable-next-line no-console + console.log('result', result) // outputs 7 + return pool.terminate() + }) + .catch(function (err) { + // eslint-disable-next-line no-console + console.error(err) + process.exit(1) + }) + .then(() => { + process.exit(0) + }) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 930ac2aed6c..34654182ddd 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -71,6 +71,7 @@ module.exports = { 'microgateway-core': () => require('../microgateway-core'), mocha: () => require('../mocha'), 'mocha-each': () => require('../mocha'), + workerpool: () => require('../mocha'), moleculer: () => require('../moleculer'), mongodb: () => require('../mongodb'), 'mongodb-core': () => require('../mongodb-core'), diff --git a/packages/datadog-instrumentations/src/mocha.js b/packages/datadog-instrumentations/src/mocha.js index 6e26b61c145..1c6998a7afc 100644 --- a/packages/datadog-instrumentations/src/mocha.js +++ b/packages/datadog-instrumentations/src/mocha.js @@ -1,674 +1,5 @@ -const { createCoverageMap } = require('istanbul-lib-coverage') - -const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') - -const { addHook, channel, AsyncResource } = require('./helpers/instrument') -const shimmer = require('../../datadog-shimmer') -const log = require('../../dd-trace/src/log') -const { - getCoveredFilenamesFromCoverage, - resetCoverage, - mergeCoverage, - getTestSuitePath, - fromCoverageMapToCoverage, - getCallSites, - addEfdStringToTestName, - removeEfdStringFromTestName -} = require('../../dd-trace/src/plugins/util/test') - -const testStartCh = channel('ci:mocha:test:start') -const errorCh = channel('ci:mocha:test:error') -const skipCh = channel('ci:mocha:test:skip') -const testFinishCh = channel('ci:mocha:test:finish') -const parameterizedTestCh = channel('ci:mocha:test:parameterize') - -const libraryConfigurationCh = channel('ci:mocha:library-configuration') -const knownTestsCh = channel('ci:mocha:known-tests') -const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') - -const testSessionStartCh = channel('ci:mocha:session:start') -const testSessionFinishCh = channel('ci:mocha:session:finish') - -const testSuiteStartCh = channel('ci:mocha:test-suite:start') -const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') -const testSuiteErrorCh = channel('ci:mocha:test-suite:error') -const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') - -const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') - -// TODO: remove when root hooks and fixtures are implemented -const patched = new WeakSet() - -const testToAr = new WeakMap() -const originalFns = new WeakMap() -const testFileToSuiteAr = new Map() -const testToStartLine = new WeakMap() -const newTests = {} - -// `isWorker` is true if it's a Mocha worker -let isWorker = false - -// We'll preserve the original coverage here -const originalCoverageMap = createCoverageMap() - -let suitesToSkip = [] -let frameworkVersion -let isSuitesSkipped = false -let skippedSuites = [] -const unskippableSuites = [] -let isForcedToRun = false -let itrCorrelationId = '' -let isEarlyFlakeDetectionEnabled = false -let earlyFlakeDetectionNumRetries = 0 -let isSuitesSkippingEnabled = false -let knownTests = [] - -function getSuitesByTestFile (root) { - const suitesByTestFile = {} - function getSuites (suite) { - if (suite.file) { - if (suitesByTestFile[suite.file]) { - suitesByTestFile[suite.file].push(suite) - } else { - suitesByTestFile[suite.file] = [suite] - } - } - suite.suites.forEach(suite => { - getSuites(suite) - }) - } - getSuites(root) - - const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { - acc[testFile] = suitesByTestFile[testFile].length - return acc - }, {}) - - return { suitesByTestFile, numSuitesByTestFile } +if (process.env.MOCHA_WORKER_ID) { + require('./mocha/worker') +} else { + require('./mocha/main') } - -function getTestStatus (test) { - if (test.isPending()) { - return 'skip' - } - if (test.isFailed() || test.timedOut) { - return 'fail' - } - return 'pass' -} - -function isRetry (test) { - return test._currentRetry !== undefined && test._currentRetry !== 0 -} - -function getTestFullName (test) { - return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` -} - -function isNewTest (test) { - const testSuite = getTestSuitePath(test.file, process.cwd()) - const testName = removeEfdStringFromTestName(test.fullTitle()) - const testsForSuite = knownTests.mocha?.[testSuite] || [] - return !testsForSuite.includes(testName) -} - -function retryTest (test) { - const originalTestName = test.title - const suite = test.parent - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - const clonedTest = test.clone() - clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) - suite.addTest(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true - } -} - -function getTestAsyncResource (test) { - if (!test.fn) { - return testToAr.get(test) - } - if (!test.fn.asyncResource) { - return testToAr.get(test.fn) - } - const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) -} - -function getFilteredSuites (originalSuites) { - return originalSuites.reduce((acc, suite) => { - const testPath = getTestSuitePath(suite.file, process.cwd()) - const shouldSkip = suitesToSkip.includes(testPath) - const isUnskippable = unskippableSuites.includes(suite.file) - if (shouldSkip && !isUnskippable) { - acc.skippedSuites.add(testPath) - } else { - acc.suitesToRun.push(suite) - } - return acc - }, { suitesToRun: [], skippedSuites: new Set() }) -} - -function mochaHook (Runner) { - if (patched.has(Runner)) return Runner - - patched.add(Runner) - - shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) { - if (isEarlyFlakeDetectionEnabled) { - // by the time we reach `this.on('test')`, it is too late. We need to add retries here - suite.tests.forEach(test => { - if (!test.isPending() && isNewTest(test)) { - test._ddIsNew = true - retryTest(test) - } - }) - } - return runTests.apply(this, arguments) - }) - - shimmer.wrap(Runner.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers || isWorker) { - return run.apply(this, arguments) - } - - const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) - - const testRunAsyncResource = new AsyncResource('bound-anonymous-fn') - - this.once('end', testRunAsyncResource.bind(function () { - let status = 'pass' - let error - if (this.stats) { - status = this.stats.failures === 0 ? 'pass' : 'fail' - if (this.stats.tests === 0) { - status = 'skip' - } - } else if (this.failures !== 0) { - status = 'fail' - } - - if (isEarlyFlakeDetectionEnabled) { - /** - * If Early Flake Detection (EFD) is enabled the logic is as follows: - * - If all attempts for a test are failing, the test has failed and we will let the test process fail. - * - If just a single attempt passes, we will prevent the test process from failing. - * The rationale behind is the following: you may still be able to block your CI pipeline by gating - * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. - */ - for (const tests of Object.values(newTests)) { - const failingNewTests = tests.filter(test => test.isFailed()) - const areAllNewTestsFailing = failingNewTests.length === tests.length - if (failingNewTests.length && !areAllNewTestsFailing) { - this.stats.failures -= failingNewTests.length - this.failures -= failingNewTests.length - } - } - } - - if (status === 'fail') { - error = new Error(`Failed tests: ${this.failures}.`) - } - - testFileToSuiteAr.clear() - - let testCodeCoverageLinesTotal - if (global.__coverage__) { - try { - testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct - } catch (e) { - // ignore errors - } - // restore the original coverage - global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) - } - - testSessionFinishCh.publish({ - status, - isSuitesSkipped, - testCodeCoverageLinesTotal, - numSkippedSuites: skippedSuites.length, - hasForcedToRunSuites: isForcedToRun, - hasUnskippableSuites: !!unskippableSuites.length, - error, - isEarlyFlakeDetectionEnabled - }) - })) - - this.once('start', testRunAsyncResource.bind(function () { - const processArgv = process.argv.slice(2).join(' ') - const command = `mocha ${processArgv}` - testSessionStartCh.publish({ command, frameworkVersion }) - if (skippedSuites.length) { - itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) - } - })) - - this.on('suite', function (suite) { - if (suite.root || !suite.tests.length) { - return - } - let asyncResource = testFileToSuiteAr.get(suite.file) - if (!asyncResource) { - asyncResource = new AsyncResource('bound-anonymous-fn') - testFileToSuiteAr.set(suite.file, asyncResource) - const isUnskippable = unskippableSuites.includes(suite.file) - isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) - asyncResource.runInAsyncScope(() => { - testSuiteStartCh.publish({ - testSuite: suite.file, - isUnskippable, - isForcedToRun, - itrCorrelationId - }) - }) - } - }) - - this.on('suite end', function (suite) { - if (suite.root) { - return - } - const suitesInTestFile = suitesByTestFile[suite.file] - - const isLastSuite = --numSuitesByTestFile[suite.file] === 0 - if (!isLastSuite) { - return - } - - let status = 'pass' - if (suitesInTestFile.every(suite => suite.pending)) { - status = 'skip' - } else { - // has to check every test in the test file - suitesInTestFile.forEach(suite => { - suite.eachTest(test => { - if (test.state === 'failed' || test.timedOut) { - status = 'fail' - } - }) - }) - } - - if (global.__coverage__) { - const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) - - testSuiteCodeCoverageCh.publish({ - coverageFiles, - suiteFile: suite.file - }) - // We need to reset coverage to get a code coverage per suite - // Before that, we preserve the original coverage - mergeCoverage(global.__coverage__, originalCoverageMap) - resetCoverage(global.__coverage__) - } - - const asyncResource = testFileToSuiteAr.get(suite.file) - asyncResource.runInAsyncScope(() => { - testSuiteFinishCh.publish(status) - }) - }) - - this.on('test', (test) => { - if (isRetry(test)) { - return - } - const testStartLine = testToStartLine.get(test) - const asyncResource = new AsyncResource('bound-anonymous-fn') - testToAr.set(test.fn, asyncResource) - - const { - file: testSuiteAbsolutePath, - title, - _ddIsNew: isNew, - _ddIsEfdRetry: isEfdRetry - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - isNew, - isEfdRetry, - testStartLine - } - - // We want to store the result of the new tests - if (isNew) { - const testFullName = getTestFullName(test) - if (newTests[testFullName]) { - newTests[testFullName].push(test) - } else { - newTests[testFullName] = [test] - } - } - - asyncResource.runInAsyncScope(() => { - testStartCh.publish(testInfo) - }) - }) - - this.on('test end', (test) => { - const asyncResource = getTestAsyncResource(test) - const status = getTestStatus(test) - - // if there are afterEach to be run, we don't finish the test yet - if (asyncResource && !test.parent._afterEach.length) { - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - }) - - // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted - this.on('hook end', (hook) => { - const test = hook.ctx.currentTest - if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach - const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 - if (isLastAfterEach) { - const status = getTestStatus(test) - const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - } - }) - - this.on('fail', (testOrHook, err) => { - const testFile = testOrHook.file - let test = testOrHook - const isHook = testOrHook.type === 'hook' - if (isHook && testOrHook.ctx) { - test = testOrHook.ctx.currentTest - } - let testAsyncResource - if (test) { - testAsyncResource = getTestAsyncResource(test) - } - if (testAsyncResource) { - testAsyncResource.runInAsyncScope(() => { - if (isHook) { - err.message = `${testOrHook.fullTitle()}: ${err.message}` - errorCh.publish(err) - // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') - } else { - errorCh.publish(err) - } - }) - } - const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) - - if (testSuiteAsyncResource) { - // we propagate the error to the suite - const testSuiteError = new Error( - `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` - ) - testSuiteError.stack = err.stack - testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteErrorCh.publish(testSuiteError) - }) - } - }) - - this.on('pending', (test) => { - const testStartLine = testToStartLine.get(test) - const { - file: testSuiteAbsolutePath, - title - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - testStartLine - } - - const asyncResource = getTestAsyncResource(test) - if (asyncResource) { - asyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } else { - // if there is no async resource, the test has been skipped through `test.skip` - // or the parent suite is skipped - const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') - if (test.fn) { - testToAr.set(test.fn, skippedTestAsyncResource) - } else { - testToAr.set(test, skippedTestAsyncResource) - } - skippedTestAsyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } - }) - - return run.apply(this, arguments) - }) - - return Runner -} - -function mochaEachHook (mochaEach) { - if (patched.has(mochaEach)) return mochaEach - - patched.add(mochaEach) - - return shimmer.wrap(mochaEach, function () { - const [params] = arguments - const { it, ...rest } = mochaEach.apply(this, arguments) - return { - it: function (title) { - parameterizedTestCh.publish({ title, params }) - it.apply(this, arguments) - }, - ...rest - } - }) -} - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/mocha.js' -}, (Mocha, mochaVersion) => { - frameworkVersion = mochaVersion - const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') - /** - * Get ITR configuration and skippable suites - * If ITR is disabled, `onDone` is called immediately on the subscriber - */ - shimmer.wrap(Mocha.prototype, 'run', run => function () { - if (this.options.parallel) { - log.warn('Unable to initialize CI Visibility because Mocha is running in parallel mode.') - return run.apply(this, arguments) - } - - if (!libraryConfigurationCh.hasSubscribers || this.isWorker) { - if (this.isWorker) { - isWorker = true - } - return run.apply(this, arguments) - } - this.options.delay = true - - const runner = run.apply(this, arguments) - - this.files.forEach(path => { - const isUnskippable = isMarkedAsUnskippable({ path }) - if (isUnskippable) { - unskippableSuites.push(path) - } - }) - - const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { - if (err) { - suitesToSkip = [] - } else { - suitesToSkip = skippableSuites - itrCorrelationId = responseItrCorrelationId - } - // We remove the suites that we skip through ITR - const filteredSuites = getFilteredSuites(runner.suite.suites) - const { suitesToRun } = filteredSuites - - isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length - - log.debug( - () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` - ) - - runner.suite.suites = suitesToRun - - skippedSuites = Array.from(filteredSuites.skippedSuites) - - global.run() - } - - const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => { - if (err) { - knownTests = [] - isEarlyFlakeDetectionEnabled = false - } else { - knownTests = receivedKnownTests - } - - if (isSuitesSkippingEnabled) { - skippableSuitesCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) - }) - } else { - global.run() - } - } - - const onReceivedConfiguration = ({ err, libraryConfig }) => { - if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) { - return global.run() - } - - isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled - isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled - earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries - - if (isEarlyFlakeDetectionEnabled) { - knownTestsCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) - }) - } else if (isSuitesSkippingEnabled) { - skippableSuitesCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) - }) - } else { - global.run() - } - } - - mochaRunAsyncResource.runInAsyncScope(() => { - libraryConfigurationCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) - }) - }) - return runner - }) - return Mocha -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/suite.js' -}, (Suite) => { - shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { - const callSites = getCallSites() - let startLine - const testCallSite = callSites.find(site => site.getFileName() === test.file) - if (testCallSite) { - startLine = testCallSite.getLineNumber() - testToStartLine.set(test, startLine) - } - return addTest.apply(this, arguments) - }) - return Suite -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runner.js' -}, mochaHook) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/cli/run-helpers.js' -}, (run) => { - shimmer.wrap(run, 'runMocha', runMocha => async function () { - if (!testStartCh.hasSubscribers) { - return runMocha.apply(this, arguments) - } - const mocha = arguments[0] - /** - * This attaches `run` to the global context, which we'll call after - * our configuration and skippable suites requests - */ - if (!mocha.options.parallel) { - mocha.options.delay = true - } - return runMocha.apply(this, arguments) - }) - return run -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runnable.js' -}, (Runnable) => { - shimmer.wrap(Runnable.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers) { - return run.apply(this, arguments) - } - const isBeforeEach = this.parent._beforeEach.includes(this) - const isAfterEach = this.parent._afterEach.includes(this) - - const isTestHook = isBeforeEach || isAfterEach - - // we restore the original user defined function - if (this.fn.asyncResource) { - const originalFn = originalFns.get(this.fn) - this.fn = originalFn - } - - if (isTestHook || this.type === 'test') { - const test = isTestHook ? this.ctx.currentTest : this - const asyncResource = getTestAsyncResource(test) - - if (asyncResource) { - // we bind the test fn to the correct async resource - const newFn = asyncResource.bind(this.fn) - - // we store the original function, not to lose it - originalFns.set(newFn, this.fn) - this.fn = newFn - - // Temporarily keep functionality when .asyncResource is removed from node - // in https://github.com/nodejs/node/pull/46432 - if (!this.fn.asyncResource) { - this.fn.asyncResource = asyncResource - } - } - } - - return run.apply(this, arguments) - }) - return Runnable -}) - -addHook({ - name: 'mocha-each', - versions: ['>=2.0.1'] -}, mochaEachHook) diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js new file mode 100644 index 00000000000..55eded56b63 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -0,0 +1,50 @@ +const { addHook, channel } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { getCallSites } = require('../../../dd-trace/src/plugins/util/test') + +const parameterizedTestCh = channel('ci:mocha:test:parameterize') +const patched = new WeakSet() +const testToStartLine = new WeakMap() + +// mocha-each support +addHook({ + name: 'mocha-each', + versions: ['>=2.0.1'] +}, mochaEach => { + if (patched.has(mochaEach)) return mochaEach + + patched.add(mochaEach) + + return shimmer.wrap(mochaEach, function () { + const [params] = arguments + const { it, ...rest } = mochaEach.apply(this, arguments) + return { + it: function (title) { + parameterizedTestCh.publish({ title, params }) + it.apply(this, arguments) + }, + ...rest + } + }) +}) + +// support for start line +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/suite.js' +}, (Suite) => { + shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { + const callSites = getCallSites() + let startLine + const testCallSite = callSites.find(site => site.getFileName() === test.file) + if (testCallSite) { + startLine = testCallSite.getLineNumber() + testToStartLine.set(test, startLine) + } + return addTest.apply(this, arguments) + }) + return Suite +}) + +module.exports = { testToStartLine } diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js new file mode 100644 index 00000000000..14cbf028071 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -0,0 +1,678 @@ +const { createCoverageMap } = require('istanbul-lib-coverage') +const { addHook, channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { isMarkedAsUnskippable } = require('../../../datadog-plugin-jest/src/util') +const log = require('../../../dd-trace/src/log') +const { + getTestSuitePath, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, + fromCoverageMapToCoverage, + getCoveredFilenamesFromCoverage, + mergeCoverage, + resetCoverage +} = require('../../../dd-trace/src/plugins/util/test') + +const { + isNewTest, + retryTest, + getSuitesByTestFile, + isMochaRetry, + getTestFullName, + getTestStatus +} = require('./utils') +const { testToStartLine } = require('./common') + +const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') +const patched = new WeakSet() +const newTests = {} +const testToAr = new WeakMap() +const originalFns = new WeakMap() +let suitesToSkip = [] +const unskippableSuites = [] +let isSuitesSkipped = false +let skippedSuites = [] +let isEarlyFlakeDetectionEnabled = false +let isSuitesSkippingEnabled = false +let earlyFlakeDetectionNumRetries = 0 +let knownTests = [] +let itrCorrelationId = '' +const testFileToSuiteAr = new Map() +let isForcedToRun = false + +// We'll preserve the original coverage here +const originalCoverageMap = createCoverageMap() + +// test channels +const testStartCh = channel('ci:mocha:test:start') +const testFinishCh = channel('ci:mocha:test:finish') +const errorCh = channel('ci:mocha:test:error') +const skipCh = channel('ci:mocha:test:skip') + +// test suite channels +const testSuiteStartCh = channel('ci:mocha:test-suite:start') +const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') +const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') + +// session channels +const libraryConfigurationCh = channel('ci:mocha:library-configuration') +const knownTestsCh = channel('ci:mocha:known-tests') +const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') +const workerReportTraceCh = channel('ci:mocha:worker-report:trace') +const testSessionStartCh = channel('ci:mocha:session:start') +const testSessionFinishCh = channel('ci:mocha:session:finish') +const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') + +function getFilteredSuites (originalSuites) { + return originalSuites.reduce((acc, suite) => { + const testPath = getTestSuitePath(suite.file, process.cwd()) + const shouldSkip = suitesToSkip.includes(testPath) + const isUnskippable = unskippableSuites.includes(suite.file) + if (shouldSkip && !isUnskippable) { + acc.skippedSuites.add(testPath) + } else { + acc.suitesToRun.push(suite) + } + return acc + }, { suitesToRun: [], skippedSuites: new Set() }) +} + +function getTestAsyncResource (test) { + if (!test.fn) { + return testToAr.get(test) + } + if (!test.fn.asyncResource) { + return testToAr.get(test.fn) + } + const originalFn = originalFns.get(test.fn) + return testToAr.get(originalFn) +} + +// In this hook we delay the execution with options.delay to grab library configuration, +// skippable and known tests. +// It is called but skipped in parallel mode. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/mocha.js' +}, (Mocha) => { + const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') + shimmer.wrap(Mocha.prototype, 'run', run => function () { + // Workers do not need to request any data, just run the tests + if (!testStartCh.hasSubscribers || process.env.MOCHA_WORKER_ID || this.options.parallel) { + return run.apply(this, arguments) + } + + // `options.delay` does not work in parallel mode, so ITR and EFD can't work. + // TODO: use `lib/cli/run-helpers.js#runMocha` to get the data in parallel mode. + this.options.delay = true + + const runner = run.apply(this, arguments) + + this.files.forEach(path => { + const isUnskippable = isMarkedAsUnskippable({ path }) + if (isUnskippable) { + unskippableSuites.push(path) + } + }) + + const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { + if (err) { + suitesToSkip = [] + } else { + suitesToSkip = skippableSuites + itrCorrelationId = responseItrCorrelationId + } + // We remove the suites that we skip through ITR + const filteredSuites = getFilteredSuites(runner.suite.suites) + const { suitesToRun } = filteredSuites + + isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length + + log.debug( + () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` + ) + + runner.suite.suites = suitesToRun + + skippedSuites = Array.from(filteredSuites.skippedSuites) + + global.run() + } + + const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => { + if (err) { + knownTests = [] + isEarlyFlakeDetectionEnabled = false + } else { + knownTests = receivedKnownTests + } + + if (isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + global.run() + } + } + + const onReceivedConfiguration = ({ err, libraryConfig }) => { + if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) { + return global.run() + } + + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + + if (isEarlyFlakeDetectionEnabled) { + knownTestsCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) + }) + } else if (isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + global.run() + } + } + + mochaRunAsyncResource.runInAsyncScope(() => { + libraryConfigurationCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) + }) + }) + + return runner + }) + return Mocha +}) + +// Only used to set `mocha.options.delay` to true in serial mode. When the mocha CLI is used, +// setting options.delay in Mocha#run is not enough to delay the execution. +// TODO: modify this hook to grab the data in parallel mode, so that ITR and EFD can work. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/cli/run-helpers.js' +}, (run) => { + shimmer.wrap(run, 'runMocha', runMocha => async function () { + if (!testStartCh.hasSubscribers) { + return runMocha.apply(this, arguments) + } + const mocha = arguments[0] + /** + * This attaches `run` to the global context, which we'll call after + * our configuration and skippable suites requests + */ + if (!mocha.options.parallel) { + mocha.options.delay = true + } + return runMocha.apply(this, arguments) + }) + return run +}) + +// Only used in serial mode (no --parallel flag is passed) +// This hook is used to generate session, module, suite and test events +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner, frameworkVersion) { + if (patched.has(Runner)) return Runner + + patched.add(Runner) + + shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) { + if (isEarlyFlakeDetectionEnabled) { + // by the time we reach `this.on('test')`, it is too late. We need to add retries here + suite.tests.forEach(test => { + if (!test.isPending() && isNewTest(test, knownTests)) { + test._ddIsNew = true + retryTest(test, earlyFlakeDetectionNumRetries) + } + }) + } + return runTests.apply(this, arguments) + }) + + shimmer.wrap(Runner.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + + const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) + + this.once('start', testSessionAsyncResource.bind(function () { + const processArgv = process.argv.slice(2).join(' ') + const command = `mocha ${processArgv}` + testSessionStartCh.publish({ command, frameworkVersion }) + if (skippedSuites.length) { + itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) + } + })) + + this.once('end', testSessionAsyncResource.bind(function () { + let status = 'pass' + let error + if (this.stats) { + status = this.stats.failures === 0 ? 'pass' : 'fail' + if (this.stats.tests === 0) { + status = 'skip' + } + } else if (this.failures !== 0) { + status = 'fail' + } + + if (isEarlyFlakeDetectionEnabled) { + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + for (const tests of Object.values(newTests)) { + const failingNewTests = tests.filter(test => test.isFailed()) + const areAllNewTestsFailing = failingNewTests.length === tests.length + if (failingNewTests.length && !areAllNewTestsFailing) { + this.stats.failures -= failingNewTests.length + this.failures -= failingNewTests.length + } + } + } + + if (status === 'fail') { + error = new Error(`Failed tests: ${this.failures}.`) + } + + testFileToSuiteAr.clear() + + let testCodeCoverageLinesTotal + if (global.__coverage__) { + try { + testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + // restore the original coverage + global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) + } + + testSessionFinishCh.publish({ + status, + isSuitesSkipped, + testCodeCoverageLinesTotal, + numSkippedSuites: skippedSuites.length, + hasForcedToRunSuites: isForcedToRun, + hasUnskippableSuites: !!unskippableSuites.length, + error, + isEarlyFlakeDetectionEnabled + }) + })) + + this.on('test', (test) => { + if (isMochaRetry(test)) { + return + } + const testStartLine = testToStartLine.get(test) + const asyncResource = new AsyncResource('bound-anonymous-fn') + testToAr.set(test.fn, asyncResource) + + const { + file: testSuiteAbsolutePath, + title, + _ddIsNew: isNew, + _ddIsEfdRetry: isEfdRetry + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + isNew, + isEfdRetry, + testStartLine + } + + // We want to store the result of the new tests + if (isNew) { + const testFullName = getTestFullName(test) + if (newTests[testFullName]) { + newTests[testFullName].push(test) + } else { + newTests[testFullName] = [test] + } + } + + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testInfo) + }) + }) + + this.on('test end', (test) => { + const asyncResource = getTestAsyncResource(test) + const status = getTestStatus(test) + + // if there are afterEach to be run, we don't finish the test yet + if (asyncResource && !test.parent._afterEach.length) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + }) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', (hook) => { + const test = hook.ctx.currentTest + if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach + const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (isLastAfterEach) { + const status = getTestStatus(test) + const asyncResource = getTestAsyncResource(test) + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } + }) + + this.on('fail', (testOrHook, err) => { + const testFile = testOrHook.file + let test = testOrHook + const isHook = testOrHook.type === 'hook' + if (isHook && testOrHook.ctx) { + test = testOrHook.ctx.currentTest + } + let testAsyncResource + if (test) { + testAsyncResource = getTestAsyncResource(test) + } + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + if (isHook) { + err.message = `${testOrHook.fullTitle()}: ${err.message}` + errorCh.publish(err) + // if it's a hook and it has failed, 'test end' will not be called + testFinishCh.publish('fail') + } else { + errorCh.publish(err) + } + }) + } + + const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) + + if (testSuiteAsyncResource) { + // we propagate the error to the suite + const testSuiteError = new Error( + `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` + ) + testSuiteError.stack = err.stack + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(testSuiteError) + }) + } + }) + + this.on('pending', (test) => { + const testStartLine = testToStartLine.get(test) + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } else { + // if there is no async resource, the test has been skipped through `test.skip` + // or the parent suite is skipped + const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') + if (test.fn) { + testToAr.set(test.fn, skippedTestAsyncResource) + } else { + testToAr.set(test, skippedTestAsyncResource) + } + skippedTestAsyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } + }) + + this.on('suite', function (suite) { + if (suite.root || !suite.tests.length) { + return + } + let asyncResource = testFileToSuiteAr.get(suite.file) + if (!asyncResource) { + asyncResource = new AsyncResource('bound-anonymous-fn') + testFileToSuiteAr.set(suite.file, asyncResource) + const isUnskippable = unskippableSuites.includes(suite.file) + isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) + asyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath: suite.file, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) + }) + } + }) + + this.on('suite end', function (suite) { + if (suite.root) { + return + } + const suitesInTestFile = suitesByTestFile[suite.file] + + const isLastSuite = --numSuitesByTestFile[suite.file] === 0 + if (!isLastSuite) { + return + } + + let status = 'pass' + if (suitesInTestFile.every(suite => suite.pending)) { + status = 'skip' + } else { + // has to check every test in the test file + suitesInTestFile.forEach(suite => { + suite.eachTest(test => { + if (test.state === 'failed' || test.timedOut) { + status = 'fail' + } + }) + }) + } + + if (global.__coverage__) { + const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) + + testSuiteCodeCoverageCh.publish({ + coverageFiles, + suiteFile: suite.file + }) + // We need to reset coverage to get a code coverage per suite + // Before that, we preserve the original coverage + mergeCoverage(global.__coverage__, originalCoverageMap) + resetCoverage(global.__coverage__) + } + + const asyncResource = testFileToSuiteAr.get(suite.file) + asyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + }) + + return run.apply(this, arguments) + }) + + return Runner +}) + +// Used both in serial and parallel mode: +// In serial mode, this hook is used to set the correct async resource to the test. +// In parallel mode, the same hook is executed in the worker, so this needs to be repeated in +// mocha/worker.js +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, (Runnable) => { + shimmer.wrap(Runnable.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + const isBeforeEach = this.parent._beforeEach.includes(this) + const isAfterEach = this.parent._afterEach.includes(this) + + const isTestHook = isBeforeEach || isAfterEach + + // we restore the original user defined function + if (this.fn.asyncResource) { + const originalFn = originalFns.get(this.fn) + this.fn = originalFn + } + + if (isTestHook || this.type === 'test') { + const test = isTestHook ? this.ctx.currentTest : this + const asyncResource = getTestAsyncResource(test) + + if (asyncResource) { + // we bind the test fn to the correct async resource + const newFn = asyncResource.bind(this.fn) + + // we store the original function, not to lose it + originalFns.set(newFn, this.fn) + this.fn = newFn + + // Temporarily keep functionality when .asyncResource is removed from node + // in https://github.com/nodejs/node/pull/46432 + if (!this.fn.asyncResource) { + this.fn.asyncResource = asyncResource + } + } + } + + return run.apply(this, arguments) + }) + return Runnable +}) + +// Only used in parallel mode (--parallel flag is passed) +// Used to generate suite events and receive test payloads from workers +addHook({ + name: 'workerpool', + // mocha@8.0.0 added parallel support and uses workerpool for it + // The version they use is 6.0.0: + // https://github.com/mochajs/mocha/blob/612fa31228c695f16173ac675f40ccdf26b4cfb5/package.json#L75 + versions: ['>=6.0.0'], + file: 'src/WorkerHandler.js' +}, (workerHandlerPackage) => { + shimmer.wrap(workerHandlerPackage.prototype, 'exec', exec => async function (message, [testSuiteAbsolutePath]) { + if (!testStartCh.hasSubscribers) { + return exec.apply(this, arguments) + } + this.worker.on('message', function (message) { + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === MOCHA_WORKER_TRACE_PAYLOAD_CODE) { + testSessionAsyncResource.runInAsyncScope(() => { + workerReportTraceCh.publish(payload) + }) + } + } + }) + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath + }) + }) + + const result = await exec.apply(this, arguments) + + const status = result.failureCount === 0 ? 'pass' : 'fail' + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + + return result + }) + + return workerHandlerPackage +}) + +// Only used in parallel mode (--parallel flag is passed) +// Used to start and finish test session and test module +addHook({ + name: 'mocha', + versions: ['>=8.0.0'], + file: 'lib/nodejs/parallel-buffered-runner.js' +}, (ParallelBufferedRunner, frameworkVersion) => { + shimmer.wrap(ParallelBufferedRunner.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + this.once('start', testSessionAsyncResource.bind(function () { + const processArgv = process.argv.slice(2).join(' ') + const command = `mocha ${processArgv}` + testSessionStartCh.publish({ command, frameworkVersion }) + })) + + this.once('end', testSessionAsyncResource.bind(function () { + let status = 'pass' + let error + if (this.stats) { + status = this.stats.failures === 0 ? 'pass' : 'fail' + if (this.stats.tests === 0) { + status = 'skip' + } + } else if (this.failures !== 0) { + status = 'fail' + } + + if (status === 'fail') { + error = new Error(`Failed tests: ${this.failures}.`) + } + + testFileToSuiteAr.clear() + + let testCodeCoverageLinesTotal + if (global.__coverage__) { + try { + testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + // restore the original coverage + global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) + } + + testSessionFinishCh.publish({ + status, + testCodeCoverageLinesTotal, + error, + isParallel: true + }) + })) + + return run.apply(this, arguments) + }) + + return ParallelBufferedRunner +}) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js new file mode 100644 index 00000000000..0cbc3c9a4ad --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -0,0 +1,75 @@ +const { + getTestSuitePath, + removeEfdStringFromTestName, + addEfdStringToTestName +} = require('../../../dd-trace/src/plugins/util/test') + +function isNewTest (test, knownTests) { + const testSuite = getTestSuitePath(test.file, process.cwd()) + const testName = removeEfdStringFromTestName(test.fullTitle()) + const testsForSuite = knownTests.mocha?.[testSuite] || [] + return !testsForSuite.includes(testName) +} + +function retryTest (test, earlyFlakeDetectionNumRetries) { + const originalTestName = test.title + const suite = test.parent + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + const clonedTest = test.clone() + clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + suite.addTest(clonedTest) + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } +} + +function getSuitesByTestFile (root) { + const suitesByTestFile = {} + function getSuites (suite) { + if (suite.file) { + if (suitesByTestFile[suite.file]) { + suitesByTestFile[suite.file].push(suite) + } else { + suitesByTestFile[suite.file] = [suite] + } + } + suite.suites.forEach(suite => { + getSuites(suite) + }) + } + getSuites(root) + + const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { + acc[testFile] = suitesByTestFile[testFile].length + return acc + }, {}) + + return { suitesByTestFile, numSuitesByTestFile } +} + +function isMochaRetry (test) { + return test._currentRetry !== undefined && test._currentRetry !== 0 +} + +function getTestFullName (test) { + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` +} + +function getTestStatus (test) { + if (test.isPending()) { + return 'skip' + } + if (test.isFailed() || test.timedOut) { + return 'fail' + } + return 'pass' +} + +module.exports = { + isNewTest, + retryTest, + getSuitesByTestFile, + isMochaRetry, + getTestFullName, + getTestStatus +} diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js new file mode 100644 index 00000000000..56567f1c575 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -0,0 +1,199 @@ +const { addHook, channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +const { testToStartLine } = require('./common') +const { isMochaRetry, getTestStatus } = require('./utils') + +const testToAr = new WeakMap() +const originalFns = new WeakMap() + +const testStartCh = channel('ci:mocha:test:start') +const testFinishCh = channel('ci:mocha:test:finish') +const errorCh = channel('ci:mocha:test:error') +const skipCh = channel('ci:mocha:test:skip') +const workerFinishCh = channel('ci:mocha:worker:finish') + +function getTestAsyncResource (test) { + if (!test.fn) { + return testToAr.get(test) + } + if (!test.fn.asyncResource) { + return testToAr.get(test.fn) + } + const originalFn = originalFns.get(test.fn) + return testToAr.get(originalFn) +} + +// Runner is also hooked in mocha/main.js, but in here we only generate test events. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner) { + shimmer.wrap(Runner.prototype, 'run', run => function () { + // use this chance to flush + this.on('end', () => { + workerFinishCh.publish() + }) + this.on('test', (test) => { + if (isMochaRetry(test)) { + return + } + const testStartLine = testToStartLine.get(test) + const asyncResource = new AsyncResource('bound-anonymous-fn') + testToAr.set(test.fn, asyncResource) + + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine, + isParallel: true + } + + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testInfo) + }) + }) + + this.on('test end', (test) => { + const asyncResource = getTestAsyncResource(test) + const status = getTestStatus(test) + + // if there are afterEach to be run, we don't finish the test yet + if (asyncResource && !test.parent._afterEach.length) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + }) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', (hook) => { + const test = hook.ctx.currentTest + if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach + const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (isLastAfterEach) { + const status = getTestStatus(test) + const asyncResource = getTestAsyncResource(test) + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } + }) + + this.on('fail', (testOrHook, err) => { + let test = testOrHook + const isHook = testOrHook.type === 'hook' + if (isHook && testOrHook.ctx) { + test = testOrHook.ctx.currentTest + } + let testAsyncResource + if (test) { + testAsyncResource = getTestAsyncResource(test) + } + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + if (isHook) { + err.message = `${testOrHook.fullTitle()}: ${err.message}` + errorCh.publish(err) + // if it's a hook and it has failed, 'test end' will not be called + testFinishCh.publish('fail') + } else { + errorCh.publish(err) + } + }) + } + }) + + this.on('pending', (test) => { + const testStartLine = testToStartLine.get(test) + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } else { + // if there is no async resource, the test has been skipped through `test.skip` + // or the parent suite is skipped + const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') + if (test.fn) { + testToAr.set(test.fn, skippedTestAsyncResource) + } else { + testToAr.set(test, skippedTestAsyncResource) + } + skippedTestAsyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } + }) + + return run.apply(this, arguments) + }) + return Runner +}) + +// This hook also appears in mocha/main.js +// Hook to bind the test function to the correct async resource +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, (Runnable) => { + shimmer.wrap(Runnable.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + const isBeforeEach = this.parent._beforeEach.includes(this) + const isAfterEach = this.parent._afterEach.includes(this) + + const isTestHook = isBeforeEach || isAfterEach + + // we restore the original user defined function + if (this.fn.asyncResource) { + const originalFn = originalFns.get(this.fn) + this.fn = originalFn + } + + if (isTestHook || this.type === 'test') { + const test = isTestHook ? this.ctx.currentTest : this + const asyncResource = getTestAsyncResource(test) + + if (asyncResource) { + // we bind the test fn to the correct async resource + const newFn = asyncResource.bind(this.fn) + + // we store the original function, not to lose it + originalFns.set(newFn, this.fn) + this.fn = newFn + + // Temporarily keep functionality when .asyncResource is removed from node + // in https://github.com/nodejs/node/pull/46432 + if (!this.fn.asyncResource) { + this.fn.asyncResource = asyncResource + } + } + } + + return run.apply(this, arguments) + }) + return Runnable +}) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 597dc7c9cc2..dbe311b9bf1 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -20,7 +20,14 @@ const { removeEfdStringFromTestName, TEST_IS_NEW, TEST_IS_RETRY, - TEST_EARLY_FLAKE_ENABLED + TEST_EARLY_FLAKE_ENABLED, + TEST_SESSION_ID, + TEST_MODULE_ID, + TEST_MODULE, + TEST_SUITE_ID, + TEST_COMMAND, + TEST_SUITE, + MOCHA_IS_PARALLEL } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -33,6 +40,22 @@ const { TELEMETRY_ITR_UNSKIPPABLE, TELEMETRY_CODE_COVERAGE_NUM_FILES } = require('../../dd-trace/src/ci-visibility/telemetry') +const id = require('../../dd-trace/src/id') +const log = require('../../dd-trace/src/log') + +function getTestSuiteLevelVisibilityTags (testSuiteSpan) { + const testSuiteSpanContext = testSuiteSpan.context() + const suiteTags = { + [TEST_SUITE_ID]: testSuiteSpanContext.toSpanId(), + [TEST_SESSION_ID]: testSuiteSpanContext.toTraceId(), + [TEST_COMMAND]: testSuiteSpanContext._tags[TEST_COMMAND], + [TEST_MODULE]: 'mocha' + } + if (testSuiteSpanContext._parentId) { + suiteTags[TEST_MODULE_ID] = testSuiteSpanContext._parentId.toString(10) + } + return suiteTags +} class MochaPlugin extends CiPlugin { static get id () { @@ -50,7 +73,8 @@ class MochaPlugin extends CiPlugin { if (!this.libraryConfig?.isCodeCoverageEnabled) { return } - const testSuiteSpan = this._testSuites.get(suiteFile) + const testSuite = getTestSuitePath(suiteFile, this.sourceRoot) + const testSuiteSpan = this._testSuites.get(testSuite) if (!coverageFiles.length) { this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) @@ -73,16 +97,20 @@ class MochaPlugin extends CiPlugin { }) this.addSub('ci:mocha:test-suite:start', ({ - testSuite, + testSuiteAbsolutePath, isUnskippable, isForcedToRun, itrCorrelationId }) => { - const store = storage.getStore() + // If the test module span is undefined, the plugin has not been initialized correctly and we bail out + if (!this.testModuleSpan) { + return + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) const testSuiteMetadata = getTestSuiteCommonTags( this.command, this.frameworkVersion, - getTestSuitePath(testSuite, this.sourceRoot), + testSuite, 'mocha' ) if (isUnskippable) { @@ -109,6 +137,7 @@ class MochaPlugin extends CiPlugin { if (itrCorrelationId) { testSuiteSpan.setTag(ITR_CORRELATION_ID, itrCorrelationId) } + const store = storage.getStore() this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) }) @@ -142,6 +171,10 @@ class MochaPlugin extends CiPlugin { this.enter(span, store) }) + this.addSub('ci:mocha:worker:finish', () => { + this.tracer._exporter.flush() + }) + this.addSub('ci:mocha:test:finish', (status) => { const store = storage.getStore() const span = store?.span @@ -194,7 +227,8 @@ class MochaPlugin extends CiPlugin { hasForcedToRunSuites, hasUnskippableSuites, error, - isEarlyFlakeDetectionEnabled + isEarlyFlakeDetectionEnabled, + isParallel }) => { if (this.testSessionSpan) { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} @@ -206,6 +240,10 @@ class MochaPlugin extends CiPlugin { this.testModuleSpan.setTag('error', error) } + if (isParallel) { + this.testSessionSpan.setTag(MOCHA_IS_PARALLEL, 'true') + } + addIntelligentTestRunnerSpanTags( this.testSessionSpan, this.testModuleSpan, @@ -234,6 +272,37 @@ class MochaPlugin extends CiPlugin { this.libraryConfig = null this.tracer._exporter.flush() }) + + this.addSub('ci:mocha:worker-report:trace', (traces) => { + const formattedTraces = JSON.parse(traces).map(trace => + trace.map(span => { + const formattedSpan = { + ...span, + span_id: id(span.span_id), + trace_id: id(span.trace_id), + parent_id: id(span.parent_id) + } + if (formattedSpan.name === 'mocha.test') { + const testSuite = span.meta[TEST_SUITE] + const testSuiteSpan = this._testSuites.get(testSuite) + if (!testSuiteSpan) { + log.warn(`Test suite span not found for test span with test suite ${testSuite}`) + return formattedSpan + } + const suiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan) + formattedSpan.meta = { + ...formattedSpan.meta, + ...suiteTags + } + } + return formattedSpan + }) + ) + + formattedTraces.forEach(trace => { + this.tracer._exporter.export(trace) + }) + }) } startTestSpan (testInfo) { @@ -242,7 +311,8 @@ class MochaPlugin extends CiPlugin { title, isNew, isEfdRetry, - testStartLine + testStartLine, + isParallel } = testInfo const testName = removeEfdStringFromTestName(testInfo.testName) @@ -257,8 +327,12 @@ class MochaPlugin extends CiPlugin { extraTags[TEST_SOURCE_START] = testStartLine } + if (isParallel) { + extraTags[MOCHA_IS_PARALLEL] = 'true' + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) - const testSuiteSpan = this._testSuites.get(testSuiteAbsolutePath) + const testSuiteSpan = this._testSuites.get(testSuite) if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) { extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js index f91bdd52090..e74869dbe82 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js @@ -4,7 +4,8 @@ const Writer = require('./writer') const { JEST_WORKER_COVERAGE_PAYLOAD_CODE, JEST_WORKER_TRACE_PAYLOAD_CODE, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE } = require('../../../plugins/util/test') function getInterprocessTraceCode () { @@ -14,6 +15,9 @@ function getInterprocessTraceCode () { if (process.env.CUCUMBER_WORKER_ID) { return CUCUMBER_WORKER_TRACE_PAYLOAD_CODE } + if (process.env.MOCHA_WORKER_ID) { + return MOCHA_WORKER_TRACE_PAYLOAD_CODE + } return null } diff --git a/packages/dd-trace/src/exporter.js b/packages/dd-trace/src/exporter.js index 01bb96ac380..02d50c3b57e 100644 --- a/packages/dd-trace/src/exporter.js +++ b/packages/dd-trace/src/exporter.js @@ -19,6 +19,7 @@ module.exports = name => { return require('./ci-visibility/exporters/agent-proxy') case exporters.JEST_WORKER: case exporters.CUCUMBER_WORKER: + case exporters.MOCHA_WORKER: return require('./ci-visibility/exporters/test-worker') default: return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent') diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 82217ada440..d7193917b05 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -54,6 +54,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') }, get 'mongodb-core' () { return require('../../../datadog-plugin-mongodb-core/src') }, diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 23ce067670a..d1d1861ea5d 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -62,6 +62,7 @@ const JEST_TEST_RUNNER = 'test.jest.test_runner' const JEST_DISPLAY_NAME = 'test.jest.display_name' const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel' +const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel' const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped' const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled' @@ -87,6 +88,9 @@ const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61 // cucumber worker variables const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70 +// mocha worker variables +const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80 + // Early flake detection util strings const EFD_STRING = "Retried by Datadog's Early Flake Detection" const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') @@ -98,6 +102,7 @@ module.exports = { JEST_TEST_RUNNER, JEST_DISPLAY_NAME, CUCUMBER_IS_PARALLEL, + MOCHA_IS_PARALLEL, TEST_TYPE, TEST_NAME, TEST_SUITE, @@ -111,6 +116,7 @@ module.exports = { JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, TEST_SOURCE_START, TEST_SKIPPED_BY_ITR, TEST_CONFIGURATION_BROWSER_NAME, diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js index cb212eca891..bc9d56eea99 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -6,16 +6,19 @@ const TestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/ex const { JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE } = require('../../../../src/plugins/util/test') describe('CI Visibility Test Worker Exporter', () => { let send, originalSend + beforeEach(() => { send = sinon.spy() originalSend = process.send process.send = send }) + afterEach(() => { process.send = originalSend }) @@ -80,4 +83,29 @@ describe('CI Visibility Test Worker Exporter', () => { expect(send).not.to.have.been.called }) }) + context('when the process is a MOCHA worker', () => { + beforeEach(() => { + process.env.MOCHA_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.MOCHA_WORKER_ID + }) + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.export(traceSecond) + mochaWorkerExporter.flush() + expect(send).to.have.been.calledWith([MOCHA_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) }) From c5671a079f1b5e85250dca12a70db5d94dcb864b Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Wed, 22 May 2024 11:55:02 +0200 Subject: [PATCH 2/5] extract common code and handle promise rejections in worker hook --- .../src/mocha/common.js | 2 +- .../src/mocha/main.js | 425 +++++------------- .../src/mocha/utils.js | 232 +++++++++- .../src/mocha/worker.js | 187 +------- 4 files changed, 364 insertions(+), 482 deletions(-) diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js index 55eded56b63..11b9015c93b 100644 --- a/packages/datadog-instrumentations/src/mocha/common.js +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -1,10 +1,10 @@ const { addHook, channel } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') const { getCallSites } = require('../../../dd-trace/src/plugins/util/test') +const { testToStartLine } = require('./utils') const parameterizedTestCh = channel('ci:mocha:test:parameterize') const patched = new WeakSet() -const testToStartLine = new WeakMap() // mocha-each support addHook({ diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 14cbf028071..6139da7ceaa 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -16,17 +16,17 @@ const { isNewTest, retryTest, getSuitesByTestFile, - isMochaRetry, - getTestFullName, - getTestStatus + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler } = require('./utils') -const { testToStartLine } = require('./common') const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') const patched = new WeakSet() const newTests = {} -const testToAr = new WeakMap() -const originalFns = new WeakMap() let suitesToSkip = [] const unskippableSuites = [] let isSuitesSkipped = false @@ -44,9 +44,6 @@ const originalCoverageMap = createCoverageMap() // test channels const testStartCh = channel('ci:mocha:test:start') -const testFinishCh = channel('ci:mocha:test:finish') -const errorCh = channel('ci:mocha:test:error') -const skipCh = channel('ci:mocha:test:skip') // test suite channels const testSuiteStartCh = channel('ci:mocha:test-suite:start') @@ -77,15 +74,77 @@ function getFilteredSuites (originalSuites) { }, { suitesToRun: [], skippedSuites: new Set() }) } -function getTestAsyncResource (test) { - if (!test.fn) { - return testToAr.get(test) - } - if (!test.fn.asyncResource) { - return testToAr.get(test.fn) - } - const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) +function getOnStartHandler (isParallel, frameworkVersion) { + return testSessionAsyncResource.bind(function () { + const processArgv = process.argv.slice(2).join(' ') + const command = `mocha ${processArgv}` + testSessionStartCh.publish({ command, frameworkVersion }) + if (!isParallel && skippedSuites.length) { + itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) + } + }) +} + +function getOnEndHandler (isParallel) { + return testSessionAsyncResource.bind(function () { + let status = 'pass' + let error + if (this.stats) { + status = this.stats.failures === 0 ? 'pass' : 'fail' + if (this.stats.tests === 0) { + status = 'skip' + } + } else if (this.failures !== 0) { + status = 'fail' + } + + if (!isParallel && isEarlyFlakeDetectionEnabled) { + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + for (const tests of Object.values(newTests)) { + const failingNewTests = tests.filter(test => test.isFailed()) + const areAllNewTestsFailing = failingNewTests.length === tests.length + if (failingNewTests.length && !areAllNewTestsFailing) { + this.stats.failures -= failingNewTests.length + this.failures -= failingNewTests.length + } + } + } + + if (status === 'fail') { + error = new Error(`Failed tests: ${this.failures}.`) + } + + testFileToSuiteAr.clear() + + let testCodeCoverageLinesTotal + if (global.__coverage__) { + try { + testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + // restore the original coverage + global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) + } + + testSessionFinishCh.publish({ + status, + isSuitesSkipped, + testCodeCoverageLinesTotal, + numSkippedSuites: skippedSuites.length, + hasForcedToRunSuites: isForcedToRun, + hasUnskippableSuites: !!unskippableSuites.length, + error, + isEarlyFlakeDetectionEnabled, + isParallel + }) + }) } // In this hook we delay the execution with options.delay to grab library configuration, @@ -246,211 +305,20 @@ addHook({ const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) - this.once('start', testSessionAsyncResource.bind(function () { - const processArgv = process.argv.slice(2).join(' ') - const command = `mocha ${processArgv}` - testSessionStartCh.publish({ command, frameworkVersion }) - if (skippedSuites.length) { - itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) - } - })) - - this.once('end', testSessionAsyncResource.bind(function () { - let status = 'pass' - let error - if (this.stats) { - status = this.stats.failures === 0 ? 'pass' : 'fail' - if (this.stats.tests === 0) { - status = 'skip' - } - } else if (this.failures !== 0) { - status = 'fail' - } + this.once('start', getOnStartHandler(false, frameworkVersion)) - if (isEarlyFlakeDetectionEnabled) { - /** - * If Early Flake Detection (EFD) is enabled the logic is as follows: - * - If all attempts for a test are failing, the test has failed and we will let the test process fail. - * - If just a single attempt passes, we will prevent the test process from failing. - * The rationale behind is the following: you may still be able to block your CI pipeline by gating - * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. - */ - for (const tests of Object.values(newTests)) { - const failingNewTests = tests.filter(test => test.isFailed()) - const areAllNewTestsFailing = failingNewTests.length === tests.length - if (failingNewTests.length && !areAllNewTestsFailing) { - this.stats.failures -= failingNewTests.length - this.failures -= failingNewTests.length - } - } - } + this.once('end', getOnEndHandler(false)) - if (status === 'fail') { - error = new Error(`Failed tests: ${this.failures}.`) - } + this.on('test', getOnTestHandler(true, newTests)) - testFileToSuiteAr.clear() - - let testCodeCoverageLinesTotal - if (global.__coverage__) { - try { - testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct - } catch (e) { - // ignore errors - } - // restore the original coverage - global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) - } - - testSessionFinishCh.publish({ - status, - isSuitesSkipped, - testCodeCoverageLinesTotal, - numSkippedSuites: skippedSuites.length, - hasForcedToRunSuites: isForcedToRun, - hasUnskippableSuites: !!unskippableSuites.length, - error, - isEarlyFlakeDetectionEnabled - }) - })) - - this.on('test', (test) => { - if (isMochaRetry(test)) { - return - } - const testStartLine = testToStartLine.get(test) - const asyncResource = new AsyncResource('bound-anonymous-fn') - testToAr.set(test.fn, asyncResource) - - const { - file: testSuiteAbsolutePath, - title, - _ddIsNew: isNew, - _ddIsEfdRetry: isEfdRetry - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - isNew, - isEfdRetry, - testStartLine - } - - // We want to store the result of the new tests - if (isNew) { - const testFullName = getTestFullName(test) - if (newTests[testFullName]) { - newTests[testFullName].push(test) - } else { - newTests[testFullName] = [test] - } - } - - asyncResource.runInAsyncScope(() => { - testStartCh.publish(testInfo) - }) - }) - - this.on('test end', (test) => { - const asyncResource = getTestAsyncResource(test) - const status = getTestStatus(test) - - // if there are afterEach to be run, we don't finish the test yet - if (asyncResource && !test.parent._afterEach.length) { - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - }) + this.on('test end', getOnTestEndHandler()) // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted - this.on('hook end', (hook) => { - const test = hook.ctx.currentTest - if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach - const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 - if (isLastAfterEach) { - const status = getTestStatus(test) - const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - } - }) - - this.on('fail', (testOrHook, err) => { - const testFile = testOrHook.file - let test = testOrHook - const isHook = testOrHook.type === 'hook' - if (isHook && testOrHook.ctx) { - test = testOrHook.ctx.currentTest - } - let testAsyncResource - if (test) { - testAsyncResource = getTestAsyncResource(test) - } - if (testAsyncResource) { - testAsyncResource.runInAsyncScope(() => { - if (isHook) { - err.message = `${testOrHook.fullTitle()}: ${err.message}` - errorCh.publish(err) - // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') - } else { - errorCh.publish(err) - } - }) - } + this.on('hook end', getOnHookEndHandler()) - const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) + this.on('fail', getOnFailHandler(true)) - if (testSuiteAsyncResource) { - // we propagate the error to the suite - const testSuiteError = new Error( - `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` - ) - testSuiteError.stack = err.stack - testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteErrorCh.publish(testSuiteError) - }) - } - }) - - this.on('pending', (test) => { - const testStartLine = testToStartLine.get(test) - const { - file: testSuiteAbsolutePath, - title - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - testStartLine - } - - const asyncResource = getTestAsyncResource(test) - if (asyncResource) { - asyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } else { - // if there is no async resource, the test has been skipped through `test.skip` - // or the parent suite is skipped - const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') - if (test.fn) { - testToAr.set(test.fn, skippedTestAsyncResource) - } else { - testToAr.set(test, skippedTestAsyncResource) - } - skippedTestAsyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } - }) + this.on('pending', getOnPendingHandler()) this.on('suite', function (suite) { if (suite.root || !suite.tests.length) { @@ -523,54 +391,13 @@ addHook({ return Runner }) -// Used both in serial and parallel mode: -// In serial mode, this hook is used to set the correct async resource to the test. -// In parallel mode, the same hook is executed in the worker, so this needs to be repeated in -// mocha/worker.js +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. addHook({ name: 'mocha', versions: ['>=5.2.0'], file: 'lib/runnable.js' -}, (Runnable) => { - shimmer.wrap(Runnable.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers) { - return run.apply(this, arguments) - } - const isBeforeEach = this.parent._beforeEach.includes(this) - const isAfterEach = this.parent._afterEach.includes(this) - - const isTestHook = isBeforeEach || isAfterEach - - // we restore the original user defined function - if (this.fn.asyncResource) { - const originalFn = originalFns.get(this.fn) - this.fn = originalFn - } - - if (isTestHook || this.type === 'test') { - const test = isTestHook ? this.ctx.currentTest : this - const asyncResource = getTestAsyncResource(test) - - if (asyncResource) { - // we bind the test fn to the correct async resource - const newFn = asyncResource.bind(this.fn) - - // we store the original function, not to lose it - originalFns.set(newFn, this.fn) - this.fn = newFn - - // Temporarily keep functionality when .asyncResource is removed from node - // in https://github.com/nodejs/node/pull/46432 - if (!this.fn.asyncResource) { - this.fn.asyncResource = asyncResource - } - } - } - - return run.apply(this, arguments) - }) - return Runnable -}) +}, runnableWrapper) // Only used in parallel mode (--parallel flag is passed) // Used to generate suite events and receive test payloads from workers @@ -582,7 +409,7 @@ addHook({ versions: ['>=6.0.0'], file: 'src/WorkerHandler.js' }, (workerHandlerPackage) => { - shimmer.wrap(workerHandlerPackage.prototype, 'exec', exec => async function (message, [testSuiteAbsolutePath]) { + shimmer.wrap(workerHandlerPackage.prototype, 'exec', exec => function (message, [testSuiteAbsolutePath]) { if (!testStartCh.hasSubscribers) { return exec.apply(this, arguments) } @@ -604,14 +431,30 @@ addHook({ }) }) - const result = await exec.apply(this, arguments) - - const status = result.failureCount === 0 ? 'pass' : 'fail' - testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteFinishCh.publish(status) - }) - - return result + try { + const promise = exec.apply(this, arguments) + promise.then( + (result) => { + const status = result.failureCount === 0 ? 'pass' : 'fail' + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + }, + (err) => { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + } + ) + return promise + } catch (err) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + throw err + } }) return workerHandlerPackage @@ -628,48 +471,8 @@ addHook({ if (!testStartCh.hasSubscribers) { return run.apply(this, arguments) } - this.once('start', testSessionAsyncResource.bind(function () { - const processArgv = process.argv.slice(2).join(' ') - const command = `mocha ${processArgv}` - testSessionStartCh.publish({ command, frameworkVersion }) - })) - - this.once('end', testSessionAsyncResource.bind(function () { - let status = 'pass' - let error - if (this.stats) { - status = this.stats.failures === 0 ? 'pass' : 'fail' - if (this.stats.tests === 0) { - status = 'skip' - } - } else if (this.failures !== 0) { - status = 'fail' - } - - if (status === 'fail') { - error = new Error(`Failed tests: ${this.failures}.`) - } - - testFileToSuiteAr.clear() - - let testCodeCoverageLinesTotal - if (global.__coverage__) { - try { - testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct - } catch (e) { - // ignore errors - } - // restore the original coverage - global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) - } - - testSessionFinishCh.publish({ - status, - testCodeCoverageLinesTotal, - error, - isParallel: true - }) - })) + this.once('start', getOnStartHandler(true, frameworkVersion)) + this.once('end', getOnEndHandler(true)) return run.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index 0cbc3c9a4ad..cc99f48e2aa 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -3,6 +3,22 @@ const { removeEfdStringFromTestName, addEfdStringToTestName } = require('../../../dd-trace/src/plugins/util/test') +const { channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +// test channels +const testStartCh = channel('ci:mocha:test:start') +const testFinishCh = channel('ci:mocha:test:finish') +const errorCh = channel('ci:mocha:test:error') +const skipCh = channel('ci:mocha:test:skip') + +// suite channels +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') + +const testToAr = new WeakMap() +const originalFns = new WeakMap() +const testToStartLine = new WeakMap() +const testFileToSuiteAr = new Map() function isNewTest (test, knownTests) { const testSuite = getTestSuitePath(test.file, process.cwd()) @@ -65,11 +81,225 @@ function getTestStatus (test) { return 'pass' } +function getTestAsyncResource (test) { + if (!test.fn) { + return testToAr.get(test) + } + if (!test.fn.asyncResource) { + return testToAr.get(test.fn) + } + const originalFn = originalFns.get(test.fn) + return testToAr.get(originalFn) +} + +function runnableWrapper (RunnablePackage) { + shimmer.wrap(RunnablePackage.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + const isBeforeEach = this.parent._beforeEach.includes(this) + const isAfterEach = this.parent._afterEach.includes(this) + + const isTestHook = isBeforeEach || isAfterEach + + // we restore the original user defined function + if (this.fn.asyncResource) { + const originalFn = originalFns.get(this.fn) + this.fn = originalFn + } + + if (isTestHook || this.type === 'test') { + const test = isTestHook ? this.ctx.currentTest : this + const asyncResource = getTestAsyncResource(test) + + if (asyncResource) { + // we bind the test fn to the correct async resource + const newFn = asyncResource.bind(this.fn) + + // we store the original function, not to lose it + originalFns.set(newFn, this.fn) + this.fn = newFn + + // Temporarily keep functionality when .asyncResource is removed from node + // in https://github.com/nodejs/node/pull/46432 + if (!this.fn.asyncResource) { + this.fn.asyncResource = asyncResource + } + } + } + + return run.apply(this, arguments) + }) + return RunnablePackage +} + +function getOnTestHandler (isMain, newTests) { + return function (test) { + if (isMochaRetry(test)) { + return + } + const testStartLine = testToStartLine.get(test) + const asyncResource = new AsyncResource('bound-anonymous-fn') + testToAr.set(test.fn, asyncResource) + + const { + file: testSuiteAbsolutePath, + title, + _ddIsNew: isNew, + _ddIsEfdRetry: isEfdRetry + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + if (isMain) { + testInfo.isNew = isNew + testInfo.isEfdRetry = isEfdRetry + // We want to store the result of the new tests + if (isNew) { + const testFullName = getTestFullName(test) + if (newTests[testFullName]) { + newTests[testFullName].push(test) + } else { + newTests[testFullName] = [test] + } + } + } else { + testInfo.isParallel = true + } + + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testInfo) + }) + } +} + +function getOnTestEndHandler () { + return function (test) { + const asyncResource = getTestAsyncResource(test) + const status = getTestStatus(test) + + // if there are afterEach to be run, we don't finish the test yet + if (asyncResource && !test.parent._afterEach.length) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } +} + +function getOnHookEndHandler () { + return function (hook) { + const test = hook.ctx.currentTest + if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach + const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (isLastAfterEach) { + const status = getTestStatus(test) + const asyncResource = getTestAsyncResource(test) + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } + } +} + +function getOnFailHandler (isMain) { + return function (testOrHook, err) { + const testFile = testOrHook.file + let test = testOrHook + const isHook = testOrHook.type === 'hook' + if (isHook && testOrHook.ctx) { + test = testOrHook.ctx.currentTest + } + let testAsyncResource + if (test) { + testAsyncResource = getTestAsyncResource(test) + } + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + if (isHook) { + err.message = `${testOrHook.fullTitle()}: ${err.message}` + errorCh.publish(err) + // if it's a hook and it has failed, 'test end' will not be called + testFinishCh.publish('fail') + } else { + errorCh.publish(err) + } + }) + } + + if (isMain) { + const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) + + if (testSuiteAsyncResource) { + // we propagate the error to the suite + const testSuiteError = new Error( + `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` + ) + testSuiteError.stack = err.stack + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(testSuiteError) + }) + } + } + } +} + +function getOnPendingHandler () { + return function (test) { + const testStartLine = testToStartLine.get(test) + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } else { + // if there is no async resource, the test has been skipped through `test.skip` + // or the parent suite is skipped + const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') + if (test.fn) { + testToAr.set(test.fn, skippedTestAsyncResource) + } else { + testToAr.set(test, skippedTestAsyncResource) + } + skippedTestAsyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } + } +} module.exports = { isNewTest, retryTest, getSuitesByTestFile, isMochaRetry, getTestFullName, - getTestStatus + getTestStatus, + runnableWrapper, + testToAr, + originalFns, + getTestAsyncResource, + testToStartLine, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler } diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 56567f1c575..1e014ff3d0f 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -1,29 +1,17 @@ -const { addHook, channel, AsyncResource } = require('../helpers/instrument') +const { addHook, channel } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') -const { testToStartLine } = require('./common') -const { isMochaRetry, getTestStatus } = require('./utils') +const { + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler +} = require('./utils') -const testToAr = new WeakMap() -const originalFns = new WeakMap() - -const testStartCh = channel('ci:mocha:test:start') -const testFinishCh = channel('ci:mocha:test:finish') -const errorCh = channel('ci:mocha:test:error') -const skipCh = channel('ci:mocha:test:skip') const workerFinishCh = channel('ci:mocha:worker:finish') -function getTestAsyncResource (test) { - if (!test.fn) { - return testToAr.get(test) - } - if (!test.fn.asyncResource) { - return testToAr.get(test.fn) - } - const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) -} - // Runner is also hooked in mocha/main.js, but in here we only generate test events. addHook({ name: 'mocha', @@ -31,169 +19,30 @@ addHook({ file: 'lib/runner.js' }, function (Runner) { shimmer.wrap(Runner.prototype, 'run', run => function () { - // use this chance to flush + // We flush when the worker ends with its test file (a mocha instance in a worker runs a single test file) this.on('end', () => { workerFinishCh.publish() }) - this.on('test', (test) => { - if (isMochaRetry(test)) { - return - } - const testStartLine = testToStartLine.get(test) - const asyncResource = new AsyncResource('bound-anonymous-fn') - testToAr.set(test.fn, asyncResource) - - const { - file: testSuiteAbsolutePath, - title - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - testStartLine, - isParallel: true - } - - asyncResource.runInAsyncScope(() => { - testStartCh.publish(testInfo) - }) - }) + this.on('test', getOnTestHandler(false)) - this.on('test end', (test) => { - const asyncResource = getTestAsyncResource(test) - const status = getTestStatus(test) - - // if there are afterEach to be run, we don't finish the test yet - if (asyncResource && !test.parent._afterEach.length) { - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - }) + this.on('test end', getOnTestEndHandler()) // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted - this.on('hook end', (hook) => { - const test = hook.ctx.currentTest - if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach - const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 - if (isLastAfterEach) { - const status = getTestStatus(test) - const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - } - }) + this.on('hook end', getOnHookEndHandler()) - this.on('fail', (testOrHook, err) => { - let test = testOrHook - const isHook = testOrHook.type === 'hook' - if (isHook && testOrHook.ctx) { - test = testOrHook.ctx.currentTest - } - let testAsyncResource - if (test) { - testAsyncResource = getTestAsyncResource(test) - } - if (testAsyncResource) { - testAsyncResource.runInAsyncScope(() => { - if (isHook) { - err.message = `${testOrHook.fullTitle()}: ${err.message}` - errorCh.publish(err) - // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') - } else { - errorCh.publish(err) - } - }) - } - }) - - this.on('pending', (test) => { - const testStartLine = testToStartLine.get(test) - const { - file: testSuiteAbsolutePath, - title - } = test + this.on('fail', getOnFailHandler(false)) - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - testStartLine - } - - const asyncResource = getTestAsyncResource(test) - if (asyncResource) { - asyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } else { - // if there is no async resource, the test has been skipped through `test.skip` - // or the parent suite is skipped - const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') - if (test.fn) { - testToAr.set(test.fn, skippedTestAsyncResource) - } else { - testToAr.set(test, skippedTestAsyncResource) - } - skippedTestAsyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } - }) + this.on('pending', getOnPendingHandler()) return run.apply(this, arguments) }) return Runner }) -// This hook also appears in mocha/main.js -// Hook to bind the test function to the correct async resource +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. addHook({ name: 'mocha', versions: ['>=5.2.0'], file: 'lib/runnable.js' -}, (Runnable) => { - shimmer.wrap(Runnable.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers) { - return run.apply(this, arguments) - } - const isBeforeEach = this.parent._beforeEach.includes(this) - const isAfterEach = this.parent._afterEach.includes(this) - - const isTestHook = isBeforeEach || isAfterEach - - // we restore the original user defined function - if (this.fn.asyncResource) { - const originalFn = originalFns.get(this.fn) - this.fn = originalFn - } - - if (isTestHook || this.type === 'test') { - const test = isTestHook ? this.ctx.currentTest : this - const asyncResource = getTestAsyncResource(test) - - if (asyncResource) { - // we bind the test fn to the correct async resource - const newFn = asyncResource.bind(this.fn) - - // we store the original function, not to lose it - originalFns.set(newFn, this.fn) - this.fn = newFn - - // Temporarily keep functionality when .asyncResource is removed from node - // in https://github.com/nodejs/node/pull/46432 - if (!this.fn.asyncResource) { - this.fn.asyncResource = asyncResource - } - } - } - - return run.apply(this, arguments) - }) - return Runnable -}) +}, runnableWrapper) From c969e676aa6588d9f2c9ac31376de1147c9ea37d Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Wed, 22 May 2024 11:57:06 +0200 Subject: [PATCH 3/5] remove repeated test --- integration-tests/ci-visibility.spec.js | 39 ------------------------- 1 file changed, 39 deletions(-) diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js index 34e5eda8468..09db54e380a 100644 --- a/integration-tests/ci-visibility.spec.js +++ b/integration-tests/ci-visibility.spec.js @@ -2614,45 +2614,6 @@ testFrameworks.forEach(({ }) }) - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('does not skip tests if git metadata upload fails', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not request skippable') From ef9201513fd389265425e9ff5f9343d3f6cce0f2 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Wed, 22 May 2024 12:14:14 +0200 Subject: [PATCH 4/5] PR feedback --- packages/datadog-instrumentations/src/mocha/main.js | 5 +++++ packages/datadog-instrumentations/src/mocha/utils.js | 2 ++ .../datadog-instrumentations/src/mocha/worker.js | 2 ++ .../exporters/test-worker/exporter.spec.js | 12 +++++++++++- 4 files changed, 20 insertions(+), 1 deletion(-) diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 6139da7ceaa..4b4903604d1 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -1,3 +1,5 @@ +'use strict' + const { createCoverageMap } = require('istanbul-lib-coverage') const { addHook, channel, AsyncResource } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') @@ -261,6 +263,7 @@ addHook({ if (!testStartCh.hasSubscribers) { return runMocha.apply(this, arguments) } + const mocha = arguments[0] /** * This attaches `run` to the global context, which we'll call after @@ -413,6 +416,7 @@ addHook({ if (!testStartCh.hasSubscribers) { return exec.apply(this, arguments) } + this.worker.on('message', function (message) { if (Array.isArray(message)) { const [messageCode, payload] = message @@ -471,6 +475,7 @@ addHook({ if (!testStartCh.hasSubscribers) { return run.apply(this, arguments) } + this.once('start', getOnStartHandler(true, frameworkVersion)) this.once('end', getOnEndHandler(true)) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js index cc99f48e2aa..e5f36c2fb4b 100644 --- a/packages/datadog-instrumentations/src/mocha/utils.js +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -1,3 +1,5 @@ +'use strict' + const { getTestSuitePath, removeEfdStringFromTestName, diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 1e014ff3d0f..02701dc41c6 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -1,3 +1,5 @@ +'use strict' + const { addHook, channel } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js index bc9d56eea99..3322fbb8e85 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -22,6 +22,7 @@ describe('CI Visibility Test Worker Exporter', () => { afterEach(() => { process.send = originalSend }) + context('when the process is a jest worker', () => { beforeEach(() => { process.env.JEST_WORKER_ID = '1' @@ -29,6 +30,7 @@ describe('CI Visibility Test Worker Exporter', () => { afterEach(() => { delete process.env.JEST_WORKER_ID }) + it('can export traces', () => { const trace = [{ type: 'test' }] const traceSecond = [{ type: 'test', name: 'other' }] @@ -38,6 +40,7 @@ describe('CI Visibility Test Worker Exporter', () => { jestWorkerExporter.flush() expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) }) + it('can export coverages', () => { const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } @@ -49,6 +52,7 @@ describe('CI Visibility Test Worker Exporter', () => { [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] ) }) + it('does not break if process.send is undefined', () => { delete process.send const trace = [{ type: 'test' }] @@ -58,6 +62,7 @@ describe('CI Visibility Test Worker Exporter', () => { expect(send).not.to.have.been.called }) }) + context('when the process is a cucumber worker', () => { beforeEach(() => { process.env.CUCUMBER_WORKER_ID = '1' @@ -65,6 +70,7 @@ describe('CI Visibility Test Worker Exporter', () => { afterEach(() => { delete process.env.CUCUMBER_WORKER_ID }) + it('can export traces', () => { const trace = [{ type: 'test' }] const traceSecond = [{ type: 'test', name: 'other' }] @@ -74,6 +80,7 @@ describe('CI Visibility Test Worker Exporter', () => { cucumberWorkerExporter.flush() expect(send).to.have.been.calledWith([CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) }) + it('does not break if process.send is undefined', () => { delete process.send const trace = [{ type: 'test' }] @@ -83,13 +90,15 @@ describe('CI Visibility Test Worker Exporter', () => { expect(send).not.to.have.been.called }) }) - context('when the process is a MOCHA worker', () => { + + context('when the process is a mocha worker', () => { beforeEach(() => { process.env.MOCHA_WORKER_ID = '1' }) afterEach(() => { delete process.env.MOCHA_WORKER_ID }) + it('can export traces', () => { const trace = [{ type: 'test' }] const traceSecond = [{ type: 'test', name: 'other' }] @@ -99,6 +108,7 @@ describe('CI Visibility Test Worker Exporter', () => { mochaWorkerExporter.flush() expect(send).to.have.been.calledWith([MOCHA_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) }) + it('does not break if process.send is undefined', () => { delete process.send const trace = [{ type: 'test' }] From bb2c5fc3555f8683f6a71e147b91a8cd3c8fb3d9 Mon Sep 17 00:00:00 2001 From: Juan Fernandez Date: Wed, 22 May 2024 15:52:58 +0200 Subject: [PATCH 5/5] fix missing import common.js --- integration-tests/ci-visibility.spec.js | 8 ++++++-- packages/datadog-instrumentations/src/mocha/common.js | 2 -- packages/datadog-instrumentations/src/mocha/main.js | 1 + packages/datadog-instrumentations/src/mocha/worker.js | 1 + 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js index 09db54e380a..5345f8dc997 100644 --- a/integration-tests/ci-visibility.spec.js +++ b/integration-tests/ci-visibility.spec.js @@ -34,7 +34,8 @@ const { TEST_EARLY_FLAKE_ABORT_REASON, TEST_COMMAND, TEST_MODULE, - MOCHA_IS_PARALLEL + MOCHA_IS_PARALLEL, + TEST_SOURCE_START } = require('../packages/dd-trace/src/plugins/util/test') const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') @@ -90,7 +91,7 @@ testFrameworks.forEach(({ runTestsWithCoverageCommand, type }) => { - describe(`${name} ${type}`, () => { + describe.only(`${name} ${type}`, () => { let receiver let childProcess let sandbox @@ -184,6 +185,7 @@ testFrameworks.forEach(({ tests.forEach(({ meta, + metrics, test_suite_id: testSuiteId, test_module_id: testModuleId, test_session_id: testSessionId @@ -194,6 +196,7 @@ testFrameworks.forEach(({ assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + assert.exists(metrics[TEST_SOURCE_START]) }) }) @@ -1628,6 +1631,7 @@ testFrameworks.forEach(({ testSpans.forEach(testSpan => { assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) }) done() diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js index 11b9015c93b..11168c55ce2 100644 --- a/packages/datadog-instrumentations/src/mocha/common.js +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -46,5 +46,3 @@ addHook({ }) return Suite }) - -module.exports = { testToStartLine } diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js index 4b4903604d1..cb14d8999e5 100644 --- a/packages/datadog-instrumentations/src/mocha/main.js +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -25,6 +25,7 @@ const { getOnFailHandler, getOnPendingHandler } = require('./utils') +require('./common') const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') const patched = new WeakSet() diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index 02701dc41c6..c2fa26f1504 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -11,6 +11,7 @@ const { getOnFailHandler, getOnPendingHandler } = require('./utils') +require('./common') const workerFinishCh = channel('ci:mocha:worker:finish')