From ead2f410433b794f14170023e7b7cfcfb51dce3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Thu, 23 May 2024 10:03:03 +0200 Subject: [PATCH] [ci-visibility] Support mocha parallel mode (#4314) --- ci/init.js | 7 + ext/exporters.d.ts | 1 + ext/exporters.js | 3 +- integration-tests/ci-visibility.spec.js | 177 +++-- .../ci-visibility/run-workerpool.js | 23 + .../src/helpers/hooks.js | 1 + .../datadog-instrumentations/src/mocha.js | 677 +----------------- .../src/mocha/common.js | 48 ++ .../src/mocha/main.js | 487 +++++++++++++ .../src/mocha/utils.js | 307 ++++++++ .../src/mocha/worker.js | 51 ++ packages/datadog-plugin-mocha/src/index.js | 90 ++- .../exporters/test-worker/index.js | 6 +- packages/dd-trace/src/exporter.js | 1 + packages/dd-trace/src/plugins/index.js | 1 + packages/dd-trace/src/plugins/util/test.js | 6 + .../exporters/test-worker/exporter.spec.js | 40 +- 17 files changed, 1196 insertions(+), 730 deletions(-) create mode 100644 integration-tests/ci-visibility/run-workerpool.js create mode 100644 packages/datadog-instrumentations/src/mocha/common.js create mode 100644 packages/datadog-instrumentations/src/mocha/main.js create mode 100644 packages/datadog-instrumentations/src/mocha/utils.js create mode 100644 packages/datadog-instrumentations/src/mocha/worker.js diff --git a/ci/init.js b/ci/init.js index 3599b2e05f4..b54e29abd4d 100644 --- a/ci/init.js +++ b/ci/init.js @@ -4,6 +4,7 @@ const { isTrue } = require('../packages/dd-trace/src/util') const isJestWorker = !!process.env.JEST_WORKER_ID const isCucumberWorker = !!process.env.CUCUMBER_WORKER_ID +const isMochaWorker = !!process.env.MOCHA_WORKER_ID const options = { startupLogs: false, @@ -44,6 +45,12 @@ if (isCucumberWorker) { } } +if (isMochaWorker) { + options.experimental = { + exporter: 'mocha_worker' + } +} + if (shouldInit) { tracer.init(options) tracer.use('fs', false) diff --git a/ext/exporters.d.ts b/ext/exporters.d.ts index d2ebaefe267..07bc2cd29e3 100644 --- a/ext/exporters.d.ts +++ b/ext/exporters.d.ts @@ -5,6 +5,7 @@ declare const exporters: { AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', CUCUMBER_WORKER: 'cucumber_worker' + MOCHA_WORKER: 'mocha_worker' } export = exporters diff --git a/ext/exporters.js b/ext/exporters.js index b615d28f459..770116c3152 100644 --- a/ext/exporters.js +++ b/ext/exporters.js @@ -5,5 +5,6 @@ module.exports = { DATADOG: 'datadog', AGENT_PROXY: 'agent_proxy', JEST_WORKER: 'jest_worker', - CUCUMBER_WORKER: 'cucumber_worker' + CUCUMBER_WORKER: 'cucumber_worker', + MOCHA_WORKER: 'mocha_worker' } diff --git a/integration-tests/ci-visibility.spec.js b/integration-tests/ci-visibility.spec.js index 5fc2429d279..b889548ad27 100644 --- a/integration-tests/ci-visibility.spec.js +++ b/integration-tests/ci-visibility.spec.js @@ -31,7 +31,11 @@ const { TEST_EARLY_FLAKE_ENABLED, TEST_NAME, JEST_DISPLAY_NAME, - TEST_EARLY_FLAKE_ABORT_REASON + TEST_EARLY_FLAKE_ABORT_REASON, + TEST_COMMAND, + TEST_MODULE, + MOCHA_IS_PARALLEL, + TEST_SOURCE_START } = require('../packages/dd-trace/src/plugins/util/test') const { ERROR_MESSAGE } = require('../packages/dd-trace/src/constants') @@ -58,7 +62,7 @@ const testFrameworks = [ { ...mochaCommonOptions, testFile: 'ci-visibility/run-mocha.js', - dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each'], + dependencies: ['mocha', 'chai@v4', 'nyc', 'mocha-each', 'workerpool'], expectedCoverageFiles: [ 'ci-visibility/run-mocha.js', 'ci-visibility/test/sum.js', @@ -87,7 +91,7 @@ testFrameworks.forEach(({ runTestsWithCoverageCommand, type }) => { - describe(`${name} ${type}`, () => { + describe.only(`${name} ${type}`, () => { let receiver let childProcess let sandbox @@ -152,11 +156,49 @@ testFrameworks.forEach(({ }) }).timeout(50000) - it('does not init CI Visibility when running in parallel mode', (done) => { - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report tests') - done(error) - }, ({ url }) => url === '/api/v2/citestcycle', 3000).catch(() => {}) + it('works with parallel mode', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const moduleEventContent = events.find(event => event.type === 'test_module_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal( + sessionEventContent.test_session_id.toString(10), + moduleEventContent.test_session_id.toString(10) + ) + suites.forEach(({ + meta, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + }) + + tests.forEach(({ + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), moduleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), moduleEventContent.test_session_id.toString(10)) + assert.propertyVal(meta, MOCHA_IS_PARALLEL, 'true') + assert.exists(metrics[TEST_SOURCE_START]) + }) + }) childProcess = fork(testFile, { cwd, @@ -175,7 +217,65 @@ testFrameworks.forEach(({ testOutput += chunk.toString() }) childProcess.on('message', () => { - assert.include(testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.') + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('works with parallel mode when run with the cli', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const sessionEventContent = events.find(event => event.type === 'test_session_end').content + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(sessionEventContent.meta[MOCHA_IS_PARALLEL], 'true') + assert.equal(suites.length, 2) + assert.equal(tests.length, 2) + }) + childProcess = exec('mocha --parallel --jobs 2 ./ci-visibility/test/ci-visibility-test*', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + eventsPromise.then(() => { + assert.notInclude(testOutput, 'TypeError') + assert.notInclude( + testOutput, 'Unable to initialize CI Visibility because Mocha is running in parallel mode.' + ) + done() + }).catch(done) + }) + }) + + it('does not blow up when workerpool is used outside of a test', (done) => { + childProcess = exec('node ./ci-visibility/run-workerpool.js', { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + }) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', (code) => { + assert.include(testOutput, 'result 7') + assert.equal(code, 0) done() }) }) @@ -1574,6 +1674,7 @@ testFrameworks.forEach(({ testSpans.forEach(testSpan => { assert.equal(testSpan.meta[TEST_SOURCE_FILE].startsWith('ci-visibility/test/ci-visibility-test'), true) + assert.exists(testSpan.metrics[TEST_SOURCE_START]) }) done() @@ -1686,6 +1787,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('does not init if DD_API_KEY is not set', (done) => { receiver.assertMessageReceived(() => { done(new Error('Should not create spans')) @@ -1714,6 +1816,7 @@ testFrameworks.forEach(({ done() }) }) + it('can report git metadata', (done) => { const searchCommitsRequestPromise = receiver.payloadReceived( ({ url }) => url === '/api/v2/git/repository/search_commits' @@ -1745,6 +1848,7 @@ testFrameworks.forEach(({ stdio: 'pipe' }) }) + it('can report code coverage', (done) => { let testOutput const libraryConfigRequestPromise = receiver.payloadReceived( @@ -1808,6 +1912,7 @@ testFrameworks.forEach(({ done() }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ itr_enabled: false, @@ -1844,6 +1949,7 @@ testFrameworks.forEach(({ } ) }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -1905,6 +2011,7 @@ testFrameworks.forEach(({ } ) }) + it('marks the test session as skipped if every suite is skipped', (done) => { receiver.setSuitesToSkip( [ @@ -1943,6 +2050,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('does not skip tests if git metadata upload fails', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -1986,6 +2094,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.setSettings({ itr_enabled: true, @@ -2025,6 +2134,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip suites if suite is marked as unskippable', (done) => { receiver.setSuitesToSkip([ { @@ -2105,6 +2215,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { receiver.setSuitesToSkip([ { @@ -2179,6 +2290,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2213,6 +2325,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in test suites', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) @@ -2281,6 +2394,7 @@ testFrameworks.forEach(({ }) }) }) + it('reports errors in test sessions', (done) => { const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { @@ -2315,6 +2429,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('can report git metadata', (done) => { const infoRequestPromise = receiver.payloadReceived(({ url }) => url === '/info') const searchCommitsRequestPromise = receiver.payloadReceived( @@ -2354,6 +2469,7 @@ testFrameworks.forEach(({ stdio: 'pipe' }) }) + it('can report code coverage', (done) => { let testOutput const libraryConfigRequestPromise = receiver.payloadReceived( @@ -2418,6 +2534,7 @@ testFrameworks.forEach(({ done() }) }) + it('does not report code coverage if disabled by the API', (done) => { receiver.setSettings({ itr_enabled: false, @@ -2448,6 +2565,7 @@ testFrameworks.forEach(({ } ) }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2503,6 +2621,7 @@ testFrameworks.forEach(({ } ) }) + it('marks the test session as skipped if every suite is skipped', (done) => { receiver.setSuitesToSkip( [ @@ -2541,44 +2660,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) - it('marks the test session as skipped if every suite is skipped', (done) => { - receiver.setSuitesToSkip( - [ - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test.js' - } - }, - { - type: 'suite', - attributes: { - suite: 'ci-visibility/test/ci-visibility-test-2.js' - } - } - ] - ) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_STATUS, 'skip') - }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) it('does not skip tests if git metadata upload fails', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not request skippable') @@ -2615,6 +2697,7 @@ testFrameworks.forEach(({ } ) }) + it('does not skip tests if test skipping is disabled by the API', (done) => { receiver.assertPayloadReceived(() => { const error = new Error('should not request skippable') @@ -2655,6 +2738,7 @@ testFrameworks.forEach(({ } ) }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { receiver.setSuitesToSkip([{ type: 'suite', @@ -2689,6 +2773,7 @@ testFrameworks.forEach(({ }).catch(done) }) }) + it('reports itr_correlation_id in test suites', (done) => { const itrCorrelationId = '4321' receiver.setItrCorrelationId(itrCorrelationId) diff --git a/integration-tests/ci-visibility/run-workerpool.js b/integration-tests/ci-visibility/run-workerpool.js new file mode 100644 index 00000000000..8a77c9e315b --- /dev/null +++ b/integration-tests/ci-visibility/run-workerpool.js @@ -0,0 +1,23 @@ +// eslint-disable-next-line +const workerpool = require('workerpool') +const pool = workerpool.pool({ workerType: 'process' }) + +function add (a, b) { + return a + b +} + +pool + .exec(add, [3, 4]) + .then((result) => { + // eslint-disable-next-line no-console + console.log('result', result) // outputs 7 + return pool.terminate() + }) + .catch(function (err) { + // eslint-disable-next-line no-console + console.error(err) + process.exit(1) + }) + .then(() => { + process.exit(0) + }) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 930ac2aed6c..34654182ddd 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -71,6 +71,7 @@ module.exports = { 'microgateway-core': () => require('../microgateway-core'), mocha: () => require('../mocha'), 'mocha-each': () => require('../mocha'), + workerpool: () => require('../mocha'), moleculer: () => require('../moleculer'), mongodb: () => require('../mongodb'), 'mongodb-core': () => require('../mongodb-core'), diff --git a/packages/datadog-instrumentations/src/mocha.js b/packages/datadog-instrumentations/src/mocha.js index 6e26b61c145..1c6998a7afc 100644 --- a/packages/datadog-instrumentations/src/mocha.js +++ b/packages/datadog-instrumentations/src/mocha.js @@ -1,674 +1,5 @@ -const { createCoverageMap } = require('istanbul-lib-coverage') - -const { isMarkedAsUnskippable } = require('../../datadog-plugin-jest/src/util') - -const { addHook, channel, AsyncResource } = require('./helpers/instrument') -const shimmer = require('../../datadog-shimmer') -const log = require('../../dd-trace/src/log') -const { - getCoveredFilenamesFromCoverage, - resetCoverage, - mergeCoverage, - getTestSuitePath, - fromCoverageMapToCoverage, - getCallSites, - addEfdStringToTestName, - removeEfdStringFromTestName -} = require('../../dd-trace/src/plugins/util/test') - -const testStartCh = channel('ci:mocha:test:start') -const errorCh = channel('ci:mocha:test:error') -const skipCh = channel('ci:mocha:test:skip') -const testFinishCh = channel('ci:mocha:test:finish') -const parameterizedTestCh = channel('ci:mocha:test:parameterize') - -const libraryConfigurationCh = channel('ci:mocha:library-configuration') -const knownTestsCh = channel('ci:mocha:known-tests') -const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') - -const testSessionStartCh = channel('ci:mocha:session:start') -const testSessionFinishCh = channel('ci:mocha:session:finish') - -const testSuiteStartCh = channel('ci:mocha:test-suite:start') -const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') -const testSuiteErrorCh = channel('ci:mocha:test-suite:error') -const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') - -const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') - -// TODO: remove when root hooks and fixtures are implemented -const patched = new WeakSet() - -const testToAr = new WeakMap() -const originalFns = new WeakMap() -const testFileToSuiteAr = new Map() -const testToStartLine = new WeakMap() -const newTests = {} - -// `isWorker` is true if it's a Mocha worker -let isWorker = false - -// We'll preserve the original coverage here -const originalCoverageMap = createCoverageMap() - -let suitesToSkip = [] -let frameworkVersion -let isSuitesSkipped = false -let skippedSuites = [] -const unskippableSuites = [] -let isForcedToRun = false -let itrCorrelationId = '' -let isEarlyFlakeDetectionEnabled = false -let earlyFlakeDetectionNumRetries = 0 -let isSuitesSkippingEnabled = false -let knownTests = [] - -function getSuitesByTestFile (root) { - const suitesByTestFile = {} - function getSuites (suite) { - if (suite.file) { - if (suitesByTestFile[suite.file]) { - suitesByTestFile[suite.file].push(suite) - } else { - suitesByTestFile[suite.file] = [suite] - } - } - suite.suites.forEach(suite => { - getSuites(suite) - }) - } - getSuites(root) - - const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { - acc[testFile] = suitesByTestFile[testFile].length - return acc - }, {}) - - return { suitesByTestFile, numSuitesByTestFile } +if (process.env.MOCHA_WORKER_ID) { + require('./mocha/worker') +} else { + require('./mocha/main') } - -function getTestStatus (test) { - if (test.isPending()) { - return 'skip' - } - if (test.isFailed() || test.timedOut) { - return 'fail' - } - return 'pass' -} - -function isRetry (test) { - return test._currentRetry !== undefined && test._currentRetry !== 0 -} - -function getTestFullName (test) { - return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` -} - -function isNewTest (test) { - const testSuite = getTestSuitePath(test.file, process.cwd()) - const testName = removeEfdStringFromTestName(test.fullTitle()) - const testsForSuite = knownTests.mocha?.[testSuite] || [] - return !testsForSuite.includes(testName) -} - -function retryTest (test) { - const originalTestName = test.title - const suite = test.parent - for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { - const clonedTest = test.clone() - clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) - suite.addTest(clonedTest) - clonedTest._ddIsNew = true - clonedTest._ddIsEfdRetry = true - } -} - -function getTestAsyncResource (test) { - if (!test.fn) { - return testToAr.get(test) - } - if (!test.fn.asyncResource) { - return testToAr.get(test.fn) - } - const originalFn = originalFns.get(test.fn) - return testToAr.get(originalFn) -} - -function getFilteredSuites (originalSuites) { - return originalSuites.reduce((acc, suite) => { - const testPath = getTestSuitePath(suite.file, process.cwd()) - const shouldSkip = suitesToSkip.includes(testPath) - const isUnskippable = unskippableSuites.includes(suite.file) - if (shouldSkip && !isUnskippable) { - acc.skippedSuites.add(testPath) - } else { - acc.suitesToRun.push(suite) - } - return acc - }, { suitesToRun: [], skippedSuites: new Set() }) -} - -function mochaHook (Runner) { - if (patched.has(Runner)) return Runner - - patched.add(Runner) - - shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) { - if (isEarlyFlakeDetectionEnabled) { - // by the time we reach `this.on('test')`, it is too late. We need to add retries here - suite.tests.forEach(test => { - if (!test.isPending() && isNewTest(test)) { - test._ddIsNew = true - retryTest(test) - } - }) - } - return runTests.apply(this, arguments) - }) - - shimmer.wrap(Runner.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers || isWorker) { - return run.apply(this, arguments) - } - - const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) - - const testRunAsyncResource = new AsyncResource('bound-anonymous-fn') - - this.once('end', testRunAsyncResource.bind(function () { - let status = 'pass' - let error - if (this.stats) { - status = this.stats.failures === 0 ? 'pass' : 'fail' - if (this.stats.tests === 0) { - status = 'skip' - } - } else if (this.failures !== 0) { - status = 'fail' - } - - if (isEarlyFlakeDetectionEnabled) { - /** - * If Early Flake Detection (EFD) is enabled the logic is as follows: - * - If all attempts for a test are failing, the test has failed and we will let the test process fail. - * - If just a single attempt passes, we will prevent the test process from failing. - * The rationale behind is the following: you may still be able to block your CI pipeline by gating - * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. - */ - for (const tests of Object.values(newTests)) { - const failingNewTests = tests.filter(test => test.isFailed()) - const areAllNewTestsFailing = failingNewTests.length === tests.length - if (failingNewTests.length && !areAllNewTestsFailing) { - this.stats.failures -= failingNewTests.length - this.failures -= failingNewTests.length - } - } - } - - if (status === 'fail') { - error = new Error(`Failed tests: ${this.failures}.`) - } - - testFileToSuiteAr.clear() - - let testCodeCoverageLinesTotal - if (global.__coverage__) { - try { - testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct - } catch (e) { - // ignore errors - } - // restore the original coverage - global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) - } - - testSessionFinishCh.publish({ - status, - isSuitesSkipped, - testCodeCoverageLinesTotal, - numSkippedSuites: skippedSuites.length, - hasForcedToRunSuites: isForcedToRun, - hasUnskippableSuites: !!unskippableSuites.length, - error, - isEarlyFlakeDetectionEnabled - }) - })) - - this.once('start', testRunAsyncResource.bind(function () { - const processArgv = process.argv.slice(2).join(' ') - const command = `mocha ${processArgv}` - testSessionStartCh.publish({ command, frameworkVersion }) - if (skippedSuites.length) { - itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) - } - })) - - this.on('suite', function (suite) { - if (suite.root || !suite.tests.length) { - return - } - let asyncResource = testFileToSuiteAr.get(suite.file) - if (!asyncResource) { - asyncResource = new AsyncResource('bound-anonymous-fn') - testFileToSuiteAr.set(suite.file, asyncResource) - const isUnskippable = unskippableSuites.includes(suite.file) - isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) - asyncResource.runInAsyncScope(() => { - testSuiteStartCh.publish({ - testSuite: suite.file, - isUnskippable, - isForcedToRun, - itrCorrelationId - }) - }) - } - }) - - this.on('suite end', function (suite) { - if (suite.root) { - return - } - const suitesInTestFile = suitesByTestFile[suite.file] - - const isLastSuite = --numSuitesByTestFile[suite.file] === 0 - if (!isLastSuite) { - return - } - - let status = 'pass' - if (suitesInTestFile.every(suite => suite.pending)) { - status = 'skip' - } else { - // has to check every test in the test file - suitesInTestFile.forEach(suite => { - suite.eachTest(test => { - if (test.state === 'failed' || test.timedOut) { - status = 'fail' - } - }) - }) - } - - if (global.__coverage__) { - const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) - - testSuiteCodeCoverageCh.publish({ - coverageFiles, - suiteFile: suite.file - }) - // We need to reset coverage to get a code coverage per suite - // Before that, we preserve the original coverage - mergeCoverage(global.__coverage__, originalCoverageMap) - resetCoverage(global.__coverage__) - } - - const asyncResource = testFileToSuiteAr.get(suite.file) - asyncResource.runInAsyncScope(() => { - testSuiteFinishCh.publish(status) - }) - }) - - this.on('test', (test) => { - if (isRetry(test)) { - return - } - const testStartLine = testToStartLine.get(test) - const asyncResource = new AsyncResource('bound-anonymous-fn') - testToAr.set(test.fn, asyncResource) - - const { - file: testSuiteAbsolutePath, - title, - _ddIsNew: isNew, - _ddIsEfdRetry: isEfdRetry - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - isNew, - isEfdRetry, - testStartLine - } - - // We want to store the result of the new tests - if (isNew) { - const testFullName = getTestFullName(test) - if (newTests[testFullName]) { - newTests[testFullName].push(test) - } else { - newTests[testFullName] = [test] - } - } - - asyncResource.runInAsyncScope(() => { - testStartCh.publish(testInfo) - }) - }) - - this.on('test end', (test) => { - const asyncResource = getTestAsyncResource(test) - const status = getTestStatus(test) - - // if there are afterEach to be run, we don't finish the test yet - if (asyncResource && !test.parent._afterEach.length) { - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - }) - - // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted - this.on('hook end', (hook) => { - const test = hook.ctx.currentTest - if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach - const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 - if (isLastAfterEach) { - const status = getTestStatus(test) - const asyncResource = getTestAsyncResource(test) - asyncResource.runInAsyncScope(() => { - testFinishCh.publish(status) - }) - } - } - }) - - this.on('fail', (testOrHook, err) => { - const testFile = testOrHook.file - let test = testOrHook - const isHook = testOrHook.type === 'hook' - if (isHook && testOrHook.ctx) { - test = testOrHook.ctx.currentTest - } - let testAsyncResource - if (test) { - testAsyncResource = getTestAsyncResource(test) - } - if (testAsyncResource) { - testAsyncResource.runInAsyncScope(() => { - if (isHook) { - err.message = `${testOrHook.fullTitle()}: ${err.message}` - errorCh.publish(err) - // if it's a hook and it has failed, 'test end' will not be called - testFinishCh.publish('fail') - } else { - errorCh.publish(err) - } - }) - } - const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) - - if (testSuiteAsyncResource) { - // we propagate the error to the suite - const testSuiteError = new Error( - `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` - ) - testSuiteError.stack = err.stack - testSuiteAsyncResource.runInAsyncScope(() => { - testSuiteErrorCh.publish(testSuiteError) - }) - } - }) - - this.on('pending', (test) => { - const testStartLine = testToStartLine.get(test) - const { - file: testSuiteAbsolutePath, - title - } = test - - const testInfo = { - testName: test.fullTitle(), - testSuiteAbsolutePath, - title, - testStartLine - } - - const asyncResource = getTestAsyncResource(test) - if (asyncResource) { - asyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } else { - // if there is no async resource, the test has been skipped through `test.skip` - // or the parent suite is skipped - const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') - if (test.fn) { - testToAr.set(test.fn, skippedTestAsyncResource) - } else { - testToAr.set(test, skippedTestAsyncResource) - } - skippedTestAsyncResource.runInAsyncScope(() => { - skipCh.publish(testInfo) - }) - } - }) - - return run.apply(this, arguments) - }) - - return Runner -} - -function mochaEachHook (mochaEach) { - if (patched.has(mochaEach)) return mochaEach - - patched.add(mochaEach) - - return shimmer.wrap(mochaEach, function () { - const [params] = arguments - const { it, ...rest } = mochaEach.apply(this, arguments) - return { - it: function (title) { - parameterizedTestCh.publish({ title, params }) - it.apply(this, arguments) - }, - ...rest - } - }) -} - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/mocha.js' -}, (Mocha, mochaVersion) => { - frameworkVersion = mochaVersion - const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') - /** - * Get ITR configuration and skippable suites - * If ITR is disabled, `onDone` is called immediately on the subscriber - */ - shimmer.wrap(Mocha.prototype, 'run', run => function () { - if (this.options.parallel) { - log.warn('Unable to initialize CI Visibility because Mocha is running in parallel mode.') - return run.apply(this, arguments) - } - - if (!libraryConfigurationCh.hasSubscribers || this.isWorker) { - if (this.isWorker) { - isWorker = true - } - return run.apply(this, arguments) - } - this.options.delay = true - - const runner = run.apply(this, arguments) - - this.files.forEach(path => { - const isUnskippable = isMarkedAsUnskippable({ path }) - if (isUnskippable) { - unskippableSuites.push(path) - } - }) - - const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { - if (err) { - suitesToSkip = [] - } else { - suitesToSkip = skippableSuites - itrCorrelationId = responseItrCorrelationId - } - // We remove the suites that we skip through ITR - const filteredSuites = getFilteredSuites(runner.suite.suites) - const { suitesToRun } = filteredSuites - - isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length - - log.debug( - () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` - ) - - runner.suite.suites = suitesToRun - - skippedSuites = Array.from(filteredSuites.skippedSuites) - - global.run() - } - - const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => { - if (err) { - knownTests = [] - isEarlyFlakeDetectionEnabled = false - } else { - knownTests = receivedKnownTests - } - - if (isSuitesSkippingEnabled) { - skippableSuitesCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) - }) - } else { - global.run() - } - } - - const onReceivedConfiguration = ({ err, libraryConfig }) => { - if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) { - return global.run() - } - - isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled - isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled - earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries - - if (isEarlyFlakeDetectionEnabled) { - knownTestsCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) - }) - } else if (isSuitesSkippingEnabled) { - skippableSuitesCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) - }) - } else { - global.run() - } - } - - mochaRunAsyncResource.runInAsyncScope(() => { - libraryConfigurationCh.publish({ - onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) - }) - }) - return runner - }) - return Mocha -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/suite.js' -}, (Suite) => { - shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { - const callSites = getCallSites() - let startLine - const testCallSite = callSites.find(site => site.getFileName() === test.file) - if (testCallSite) { - startLine = testCallSite.getLineNumber() - testToStartLine.set(test, startLine) - } - return addTest.apply(this, arguments) - }) - return Suite -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runner.js' -}, mochaHook) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/cli/run-helpers.js' -}, (run) => { - shimmer.wrap(run, 'runMocha', runMocha => async function () { - if (!testStartCh.hasSubscribers) { - return runMocha.apply(this, arguments) - } - const mocha = arguments[0] - /** - * This attaches `run` to the global context, which we'll call after - * our configuration and skippable suites requests - */ - if (!mocha.options.parallel) { - mocha.options.delay = true - } - return runMocha.apply(this, arguments) - }) - return run -}) - -addHook({ - name: 'mocha', - versions: ['>=5.2.0'], - file: 'lib/runnable.js' -}, (Runnable) => { - shimmer.wrap(Runnable.prototype, 'run', run => function () { - if (!testStartCh.hasSubscribers) { - return run.apply(this, arguments) - } - const isBeforeEach = this.parent._beforeEach.includes(this) - const isAfterEach = this.parent._afterEach.includes(this) - - const isTestHook = isBeforeEach || isAfterEach - - // we restore the original user defined function - if (this.fn.asyncResource) { - const originalFn = originalFns.get(this.fn) - this.fn = originalFn - } - - if (isTestHook || this.type === 'test') { - const test = isTestHook ? this.ctx.currentTest : this - const asyncResource = getTestAsyncResource(test) - - if (asyncResource) { - // we bind the test fn to the correct async resource - const newFn = asyncResource.bind(this.fn) - - // we store the original function, not to lose it - originalFns.set(newFn, this.fn) - this.fn = newFn - - // Temporarily keep functionality when .asyncResource is removed from node - // in https://github.com/nodejs/node/pull/46432 - if (!this.fn.asyncResource) { - this.fn.asyncResource = asyncResource - } - } - } - - return run.apply(this, arguments) - }) - return Runnable -}) - -addHook({ - name: 'mocha-each', - versions: ['>=2.0.1'] -}, mochaEachHook) diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js new file mode 100644 index 00000000000..11168c55ce2 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -0,0 +1,48 @@ +const { addHook, channel } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { getCallSites } = require('../../../dd-trace/src/plugins/util/test') +const { testToStartLine } = require('./utils') + +const parameterizedTestCh = channel('ci:mocha:test:parameterize') +const patched = new WeakSet() + +// mocha-each support +addHook({ + name: 'mocha-each', + versions: ['>=2.0.1'] +}, mochaEach => { + if (patched.has(mochaEach)) return mochaEach + + patched.add(mochaEach) + + return shimmer.wrap(mochaEach, function () { + const [params] = arguments + const { it, ...rest } = mochaEach.apply(this, arguments) + return { + it: function (title) { + parameterizedTestCh.publish({ title, params }) + it.apply(this, arguments) + }, + ...rest + } + }) +}) + +// support for start line +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/suite.js' +}, (Suite) => { + shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { + const callSites = getCallSites() + let startLine + const testCallSite = callSites.find(site => site.getFileName() === test.file) + if (testCallSite) { + startLine = testCallSite.getLineNumber() + testToStartLine.set(test, startLine) + } + return addTest.apply(this, arguments) + }) + return Suite +}) diff --git a/packages/datadog-instrumentations/src/mocha/main.js b/packages/datadog-instrumentations/src/mocha/main.js new file mode 100644 index 00000000000..cb14d8999e5 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/main.js @@ -0,0 +1,487 @@ +'use strict' + +const { createCoverageMap } = require('istanbul-lib-coverage') +const { addHook, channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') +const { isMarkedAsUnskippable } = require('../../../datadog-plugin-jest/src/util') +const log = require('../../../dd-trace/src/log') +const { + getTestSuitePath, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, + fromCoverageMapToCoverage, + getCoveredFilenamesFromCoverage, + mergeCoverage, + resetCoverage +} = require('../../../dd-trace/src/plugins/util/test') + +const { + isNewTest, + retryTest, + getSuitesByTestFile, + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler +} = require('./utils') +require('./common') + +const testSessionAsyncResource = new AsyncResource('bound-anonymous-fn') +const patched = new WeakSet() +const newTests = {} +let suitesToSkip = [] +const unskippableSuites = [] +let isSuitesSkipped = false +let skippedSuites = [] +let isEarlyFlakeDetectionEnabled = false +let isSuitesSkippingEnabled = false +let earlyFlakeDetectionNumRetries = 0 +let knownTests = [] +let itrCorrelationId = '' +const testFileToSuiteAr = new Map() +let isForcedToRun = false + +// We'll preserve the original coverage here +const originalCoverageMap = createCoverageMap() + +// test channels +const testStartCh = channel('ci:mocha:test:start') + +// test suite channels +const testSuiteStartCh = channel('ci:mocha:test-suite:start') +const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') +const testSuiteCodeCoverageCh = channel('ci:mocha:test-suite:code-coverage') + +// session channels +const libraryConfigurationCh = channel('ci:mocha:library-configuration') +const knownTestsCh = channel('ci:mocha:known-tests') +const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') +const workerReportTraceCh = channel('ci:mocha:worker-report:trace') +const testSessionStartCh = channel('ci:mocha:session:start') +const testSessionFinishCh = channel('ci:mocha:session:finish') +const itrSkippedSuitesCh = channel('ci:mocha:itr:skipped-suites') + +function getFilteredSuites (originalSuites) { + return originalSuites.reduce((acc, suite) => { + const testPath = getTestSuitePath(suite.file, process.cwd()) + const shouldSkip = suitesToSkip.includes(testPath) + const isUnskippable = unskippableSuites.includes(suite.file) + if (shouldSkip && !isUnskippable) { + acc.skippedSuites.add(testPath) + } else { + acc.suitesToRun.push(suite) + } + return acc + }, { suitesToRun: [], skippedSuites: new Set() }) +} + +function getOnStartHandler (isParallel, frameworkVersion) { + return testSessionAsyncResource.bind(function () { + const processArgv = process.argv.slice(2).join(' ') + const command = `mocha ${processArgv}` + testSessionStartCh.publish({ command, frameworkVersion }) + if (!isParallel && skippedSuites.length) { + itrSkippedSuitesCh.publish({ skippedSuites, frameworkVersion }) + } + }) +} + +function getOnEndHandler (isParallel) { + return testSessionAsyncResource.bind(function () { + let status = 'pass' + let error + if (this.stats) { + status = this.stats.failures === 0 ? 'pass' : 'fail' + if (this.stats.tests === 0) { + status = 'skip' + } + } else if (this.failures !== 0) { + status = 'fail' + } + + if (!isParallel && isEarlyFlakeDetectionEnabled) { + /** + * If Early Flake Detection (EFD) is enabled the logic is as follows: + * - If all attempts for a test are failing, the test has failed and we will let the test process fail. + * - If just a single attempt passes, we will prevent the test process from failing. + * The rationale behind is the following: you may still be able to block your CI pipeline by gating + * on flakiness (the test will be considered flaky), but you may choose to unblock the pipeline too. + */ + for (const tests of Object.values(newTests)) { + const failingNewTests = tests.filter(test => test.isFailed()) + const areAllNewTestsFailing = failingNewTests.length === tests.length + if (failingNewTests.length && !areAllNewTestsFailing) { + this.stats.failures -= failingNewTests.length + this.failures -= failingNewTests.length + } + } + } + + if (status === 'fail') { + error = new Error(`Failed tests: ${this.failures}.`) + } + + testFileToSuiteAr.clear() + + let testCodeCoverageLinesTotal + if (global.__coverage__) { + try { + testCodeCoverageLinesTotal = originalCoverageMap.getCoverageSummary().lines.pct + } catch (e) { + // ignore errors + } + // restore the original coverage + global.__coverage__ = fromCoverageMapToCoverage(originalCoverageMap) + } + + testSessionFinishCh.publish({ + status, + isSuitesSkipped, + testCodeCoverageLinesTotal, + numSkippedSuites: skippedSuites.length, + hasForcedToRunSuites: isForcedToRun, + hasUnskippableSuites: !!unskippableSuites.length, + error, + isEarlyFlakeDetectionEnabled, + isParallel + }) + }) +} + +// In this hook we delay the execution with options.delay to grab library configuration, +// skippable and known tests. +// It is called but skipped in parallel mode. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/mocha.js' +}, (Mocha) => { + const mochaRunAsyncResource = new AsyncResource('bound-anonymous-fn') + shimmer.wrap(Mocha.prototype, 'run', run => function () { + // Workers do not need to request any data, just run the tests + if (!testStartCh.hasSubscribers || process.env.MOCHA_WORKER_ID || this.options.parallel) { + return run.apply(this, arguments) + } + + // `options.delay` does not work in parallel mode, so ITR and EFD can't work. + // TODO: use `lib/cli/run-helpers.js#runMocha` to get the data in parallel mode. + this.options.delay = true + + const runner = run.apply(this, arguments) + + this.files.forEach(path => { + const isUnskippable = isMarkedAsUnskippable({ path }) + if (isUnskippable) { + unskippableSuites.push(path) + } + }) + + const onReceivedSkippableSuites = ({ err, skippableSuites, itrCorrelationId: responseItrCorrelationId }) => { + if (err) { + suitesToSkip = [] + } else { + suitesToSkip = skippableSuites + itrCorrelationId = responseItrCorrelationId + } + // We remove the suites that we skip through ITR + const filteredSuites = getFilteredSuites(runner.suite.suites) + const { suitesToRun } = filteredSuites + + isSuitesSkipped = suitesToRun.length !== runner.suite.suites.length + + log.debug( + () => `${suitesToRun.length} out of ${runner.suite.suites.length} suites are going to run.` + ) + + runner.suite.suites = suitesToRun + + skippedSuites = Array.from(filteredSuites.skippedSuites) + + global.run() + } + + const onReceivedKnownTests = ({ err, knownTests: receivedKnownTests }) => { + if (err) { + knownTests = [] + isEarlyFlakeDetectionEnabled = false + } else { + knownTests = receivedKnownTests + } + + if (isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + global.run() + } + } + + const onReceivedConfiguration = ({ err, libraryConfig }) => { + if (err || !skippableSuitesCh.hasSubscribers || !knownTestsCh.hasSubscribers) { + return global.run() + } + + isEarlyFlakeDetectionEnabled = libraryConfig.isEarlyFlakeDetectionEnabled + isSuitesSkippingEnabled = libraryConfig.isSuitesSkippingEnabled + earlyFlakeDetectionNumRetries = libraryConfig.earlyFlakeDetectionNumRetries + + if (isEarlyFlakeDetectionEnabled) { + knownTestsCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedKnownTests) + }) + } else if (isSuitesSkippingEnabled) { + skippableSuitesCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedSkippableSuites) + }) + } else { + global.run() + } + } + + mochaRunAsyncResource.runInAsyncScope(() => { + libraryConfigurationCh.publish({ + onDone: mochaRunAsyncResource.bind(onReceivedConfiguration) + }) + }) + + return runner + }) + return Mocha +}) + +// Only used to set `mocha.options.delay` to true in serial mode. When the mocha CLI is used, +// setting options.delay in Mocha#run is not enough to delay the execution. +// TODO: modify this hook to grab the data in parallel mode, so that ITR and EFD can work. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/cli/run-helpers.js' +}, (run) => { + shimmer.wrap(run, 'runMocha', runMocha => async function () { + if (!testStartCh.hasSubscribers) { + return runMocha.apply(this, arguments) + } + + const mocha = arguments[0] + /** + * This attaches `run` to the global context, which we'll call after + * our configuration and skippable suites requests + */ + if (!mocha.options.parallel) { + mocha.options.delay = true + } + return runMocha.apply(this, arguments) + }) + return run +}) + +// Only used in serial mode (no --parallel flag is passed) +// This hook is used to generate session, module, suite and test events +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner, frameworkVersion) { + if (patched.has(Runner)) return Runner + + patched.add(Runner) + + shimmer.wrap(Runner.prototype, 'runTests', runTests => function (suite, fn) { + if (isEarlyFlakeDetectionEnabled) { + // by the time we reach `this.on('test')`, it is too late. We need to add retries here + suite.tests.forEach(test => { + if (!test.isPending() && isNewTest(test, knownTests)) { + test._ddIsNew = true + retryTest(test, earlyFlakeDetectionNumRetries) + } + }) + } + return runTests.apply(this, arguments) + }) + + shimmer.wrap(Runner.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + + const { suitesByTestFile, numSuitesByTestFile } = getSuitesByTestFile(this.suite) + + this.once('start', getOnStartHandler(false, frameworkVersion)) + + this.once('end', getOnEndHandler(false)) + + this.on('test', getOnTestHandler(true, newTests)) + + this.on('test end', getOnTestEndHandler()) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', getOnHookEndHandler()) + + this.on('fail', getOnFailHandler(true)) + + this.on('pending', getOnPendingHandler()) + + this.on('suite', function (suite) { + if (suite.root || !suite.tests.length) { + return + } + let asyncResource = testFileToSuiteAr.get(suite.file) + if (!asyncResource) { + asyncResource = new AsyncResource('bound-anonymous-fn') + testFileToSuiteAr.set(suite.file, asyncResource) + const isUnskippable = unskippableSuites.includes(suite.file) + isForcedToRun = isUnskippable && suitesToSkip.includes(getTestSuitePath(suite.file, process.cwd())) + asyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath: suite.file, + isUnskippable, + isForcedToRun, + itrCorrelationId + }) + }) + } + }) + + this.on('suite end', function (suite) { + if (suite.root) { + return + } + const suitesInTestFile = suitesByTestFile[suite.file] + + const isLastSuite = --numSuitesByTestFile[suite.file] === 0 + if (!isLastSuite) { + return + } + + let status = 'pass' + if (suitesInTestFile.every(suite => suite.pending)) { + status = 'skip' + } else { + // has to check every test in the test file + suitesInTestFile.forEach(suite => { + suite.eachTest(test => { + if (test.state === 'failed' || test.timedOut) { + status = 'fail' + } + }) + }) + } + + if (global.__coverage__) { + const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) + + testSuiteCodeCoverageCh.publish({ + coverageFiles, + suiteFile: suite.file + }) + // We need to reset coverage to get a code coverage per suite + // Before that, we preserve the original coverage + mergeCoverage(global.__coverage__, originalCoverageMap) + resetCoverage(global.__coverage__) + } + + const asyncResource = testFileToSuiteAr.get(suite.file) + asyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + }) + + return run.apply(this, arguments) + }) + + return Runner +}) + +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, runnableWrapper) + +// Only used in parallel mode (--parallel flag is passed) +// Used to generate suite events and receive test payloads from workers +addHook({ + name: 'workerpool', + // mocha@8.0.0 added parallel support and uses workerpool for it + // The version they use is 6.0.0: + // https://github.com/mochajs/mocha/blob/612fa31228c695f16173ac675f40ccdf26b4cfb5/package.json#L75 + versions: ['>=6.0.0'], + file: 'src/WorkerHandler.js' +}, (workerHandlerPackage) => { + shimmer.wrap(workerHandlerPackage.prototype, 'exec', exec => function (message, [testSuiteAbsolutePath]) { + if (!testStartCh.hasSubscribers) { + return exec.apply(this, arguments) + } + + this.worker.on('message', function (message) { + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === MOCHA_WORKER_TRACE_PAYLOAD_CODE) { + testSessionAsyncResource.runInAsyncScope(() => { + workerReportTraceCh.publish(payload) + }) + } + } + }) + + const testSuiteAsyncResource = new AsyncResource('bound-anonymous-fn') + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteStartCh.publish({ + testSuiteAbsolutePath + }) + }) + + try { + const promise = exec.apply(this, arguments) + promise.then( + (result) => { + const status = result.failureCount === 0 ? 'pass' : 'fail' + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteFinishCh.publish(status) + }) + }, + (err) => { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + } + ) + return promise + } catch (err) { + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(err) + testSuiteFinishCh.publish('fail') + }) + throw err + } + }) + + return workerHandlerPackage +}) + +// Only used in parallel mode (--parallel flag is passed) +// Used to start and finish test session and test module +addHook({ + name: 'mocha', + versions: ['>=8.0.0'], + file: 'lib/nodejs/parallel-buffered-runner.js' +}, (ParallelBufferedRunner, frameworkVersion) => { + shimmer.wrap(ParallelBufferedRunner.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + + this.once('start', getOnStartHandler(true, frameworkVersion)) + this.once('end', getOnEndHandler(true)) + + return run.apply(this, arguments) + }) + + return ParallelBufferedRunner +}) diff --git a/packages/datadog-instrumentations/src/mocha/utils.js b/packages/datadog-instrumentations/src/mocha/utils.js new file mode 100644 index 00000000000..e5f36c2fb4b --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/utils.js @@ -0,0 +1,307 @@ +'use strict' + +const { + getTestSuitePath, + removeEfdStringFromTestName, + addEfdStringToTestName +} = require('../../../dd-trace/src/plugins/util/test') +const { channel, AsyncResource } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +// test channels +const testStartCh = channel('ci:mocha:test:start') +const testFinishCh = channel('ci:mocha:test:finish') +const errorCh = channel('ci:mocha:test:error') +const skipCh = channel('ci:mocha:test:skip') + +// suite channels +const testSuiteErrorCh = channel('ci:mocha:test-suite:error') + +const testToAr = new WeakMap() +const originalFns = new WeakMap() +const testToStartLine = new WeakMap() +const testFileToSuiteAr = new Map() + +function isNewTest (test, knownTests) { + const testSuite = getTestSuitePath(test.file, process.cwd()) + const testName = removeEfdStringFromTestName(test.fullTitle()) + const testsForSuite = knownTests.mocha?.[testSuite] || [] + return !testsForSuite.includes(testName) +} + +function retryTest (test, earlyFlakeDetectionNumRetries) { + const originalTestName = test.title + const suite = test.parent + for (let retryIndex = 0; retryIndex < earlyFlakeDetectionNumRetries; retryIndex++) { + const clonedTest = test.clone() + clonedTest.title = addEfdStringToTestName(originalTestName, retryIndex + 1) + suite.addTest(clonedTest) + clonedTest._ddIsNew = true + clonedTest._ddIsEfdRetry = true + } +} + +function getSuitesByTestFile (root) { + const suitesByTestFile = {} + function getSuites (suite) { + if (suite.file) { + if (suitesByTestFile[suite.file]) { + suitesByTestFile[suite.file].push(suite) + } else { + suitesByTestFile[suite.file] = [suite] + } + } + suite.suites.forEach(suite => { + getSuites(suite) + }) + } + getSuites(root) + + const numSuitesByTestFile = Object.keys(suitesByTestFile).reduce((acc, testFile) => { + acc[testFile] = suitesByTestFile[testFile].length + return acc + }, {}) + + return { suitesByTestFile, numSuitesByTestFile } +} + +function isMochaRetry (test) { + return test._currentRetry !== undefined && test._currentRetry !== 0 +} + +function getTestFullName (test) { + return `mocha.${getTestSuitePath(test.file, process.cwd())}.${removeEfdStringFromTestName(test.fullTitle())}` +} + +function getTestStatus (test) { + if (test.isPending()) { + return 'skip' + } + if (test.isFailed() || test.timedOut) { + return 'fail' + } + return 'pass' +} + +function getTestAsyncResource (test) { + if (!test.fn) { + return testToAr.get(test) + } + if (!test.fn.asyncResource) { + return testToAr.get(test.fn) + } + const originalFn = originalFns.get(test.fn) + return testToAr.get(originalFn) +} + +function runnableWrapper (RunnablePackage) { + shimmer.wrap(RunnablePackage.prototype, 'run', run => function () { + if (!testStartCh.hasSubscribers) { + return run.apply(this, arguments) + } + const isBeforeEach = this.parent._beforeEach.includes(this) + const isAfterEach = this.parent._afterEach.includes(this) + + const isTestHook = isBeforeEach || isAfterEach + + // we restore the original user defined function + if (this.fn.asyncResource) { + const originalFn = originalFns.get(this.fn) + this.fn = originalFn + } + + if (isTestHook || this.type === 'test') { + const test = isTestHook ? this.ctx.currentTest : this + const asyncResource = getTestAsyncResource(test) + + if (asyncResource) { + // we bind the test fn to the correct async resource + const newFn = asyncResource.bind(this.fn) + + // we store the original function, not to lose it + originalFns.set(newFn, this.fn) + this.fn = newFn + + // Temporarily keep functionality when .asyncResource is removed from node + // in https://github.com/nodejs/node/pull/46432 + if (!this.fn.asyncResource) { + this.fn.asyncResource = asyncResource + } + } + } + + return run.apply(this, arguments) + }) + return RunnablePackage +} + +function getOnTestHandler (isMain, newTests) { + return function (test) { + if (isMochaRetry(test)) { + return + } + const testStartLine = testToStartLine.get(test) + const asyncResource = new AsyncResource('bound-anonymous-fn') + testToAr.set(test.fn, asyncResource) + + const { + file: testSuiteAbsolutePath, + title, + _ddIsNew: isNew, + _ddIsEfdRetry: isEfdRetry + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + if (isMain) { + testInfo.isNew = isNew + testInfo.isEfdRetry = isEfdRetry + // We want to store the result of the new tests + if (isNew) { + const testFullName = getTestFullName(test) + if (newTests[testFullName]) { + newTests[testFullName].push(test) + } else { + newTests[testFullName] = [test] + } + } + } else { + testInfo.isParallel = true + } + + asyncResource.runInAsyncScope(() => { + testStartCh.publish(testInfo) + }) + } +} + +function getOnTestEndHandler () { + return function (test) { + const asyncResource = getTestAsyncResource(test) + const status = getTestStatus(test) + + // if there are afterEach to be run, we don't finish the test yet + if (asyncResource && !test.parent._afterEach.length) { + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } +} + +function getOnHookEndHandler () { + return function (hook) { + const test = hook.ctx.currentTest + if (test && hook.parent._afterEach.includes(hook)) { // only if it's an afterEach + const isLastAfterEach = hook.parent._afterEach.indexOf(hook) === hook.parent._afterEach.length - 1 + if (isLastAfterEach) { + const status = getTestStatus(test) + const asyncResource = getTestAsyncResource(test) + asyncResource.runInAsyncScope(() => { + testFinishCh.publish(status) + }) + } + } + } +} + +function getOnFailHandler (isMain) { + return function (testOrHook, err) { + const testFile = testOrHook.file + let test = testOrHook + const isHook = testOrHook.type === 'hook' + if (isHook && testOrHook.ctx) { + test = testOrHook.ctx.currentTest + } + let testAsyncResource + if (test) { + testAsyncResource = getTestAsyncResource(test) + } + if (testAsyncResource) { + testAsyncResource.runInAsyncScope(() => { + if (isHook) { + err.message = `${testOrHook.fullTitle()}: ${err.message}` + errorCh.publish(err) + // if it's a hook and it has failed, 'test end' will not be called + testFinishCh.publish('fail') + } else { + errorCh.publish(err) + } + }) + } + + if (isMain) { + const testSuiteAsyncResource = testFileToSuiteAr.get(testFile) + + if (testSuiteAsyncResource) { + // we propagate the error to the suite + const testSuiteError = new Error( + `"${testOrHook.parent.fullTitle()}" failed with message "${err.message}"` + ) + testSuiteError.stack = err.stack + testSuiteAsyncResource.runInAsyncScope(() => { + testSuiteErrorCh.publish(testSuiteError) + }) + } + } + } +} + +function getOnPendingHandler () { + return function (test) { + const testStartLine = testToStartLine.get(test) + const { + file: testSuiteAbsolutePath, + title + } = test + + const testInfo = { + testName: test.fullTitle(), + testSuiteAbsolutePath, + title, + testStartLine + } + + const asyncResource = getTestAsyncResource(test) + if (asyncResource) { + asyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } else { + // if there is no async resource, the test has been skipped through `test.skip` + // or the parent suite is skipped + const skippedTestAsyncResource = new AsyncResource('bound-anonymous-fn') + if (test.fn) { + testToAr.set(test.fn, skippedTestAsyncResource) + } else { + testToAr.set(test, skippedTestAsyncResource) + } + skippedTestAsyncResource.runInAsyncScope(() => { + skipCh.publish(testInfo) + }) + } + } +} +module.exports = { + isNewTest, + retryTest, + getSuitesByTestFile, + isMochaRetry, + getTestFullName, + getTestStatus, + runnableWrapper, + testToAr, + originalFns, + getTestAsyncResource, + testToStartLine, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler +} diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js new file mode 100644 index 00000000000..c2fa26f1504 --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -0,0 +1,51 @@ +'use strict' + +const { addHook, channel } = require('../helpers/instrument') +const shimmer = require('../../../datadog-shimmer') + +const { + runnableWrapper, + getOnTestHandler, + getOnTestEndHandler, + getOnHookEndHandler, + getOnFailHandler, + getOnPendingHandler +} = require('./utils') +require('./common') + +const workerFinishCh = channel('ci:mocha:worker:finish') + +// Runner is also hooked in mocha/main.js, but in here we only generate test events. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runner.js' +}, function (Runner) { + shimmer.wrap(Runner.prototype, 'run', run => function () { + // We flush when the worker ends with its test file (a mocha instance in a worker runs a single test file) + this.on('end', () => { + workerFinishCh.publish() + }) + this.on('test', getOnTestHandler(false)) + + this.on('test end', getOnTestEndHandler()) + + // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted + this.on('hook end', getOnHookEndHandler()) + + this.on('fail', getOnFailHandler(false)) + + this.on('pending', getOnPendingHandler()) + + return run.apply(this, arguments) + }) + return Runner +}) + +// Used both in serial and parallel mode, and by both the main process and the workers +// Used to set the correct async resource to the test. +addHook({ + name: 'mocha', + versions: ['>=5.2.0'], + file: 'lib/runnable.js' +}, runnableWrapper) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 597dc7c9cc2..dbe311b9bf1 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -20,7 +20,14 @@ const { removeEfdStringFromTestName, TEST_IS_NEW, TEST_IS_RETRY, - TEST_EARLY_FLAKE_ENABLED + TEST_EARLY_FLAKE_ENABLED, + TEST_SESSION_ID, + TEST_MODULE_ID, + TEST_MODULE, + TEST_SUITE_ID, + TEST_COMMAND, + TEST_SUITE, + MOCHA_IS_PARALLEL } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') const { @@ -33,6 +40,22 @@ const { TELEMETRY_ITR_UNSKIPPABLE, TELEMETRY_CODE_COVERAGE_NUM_FILES } = require('../../dd-trace/src/ci-visibility/telemetry') +const id = require('../../dd-trace/src/id') +const log = require('../../dd-trace/src/log') + +function getTestSuiteLevelVisibilityTags (testSuiteSpan) { + const testSuiteSpanContext = testSuiteSpan.context() + const suiteTags = { + [TEST_SUITE_ID]: testSuiteSpanContext.toSpanId(), + [TEST_SESSION_ID]: testSuiteSpanContext.toTraceId(), + [TEST_COMMAND]: testSuiteSpanContext._tags[TEST_COMMAND], + [TEST_MODULE]: 'mocha' + } + if (testSuiteSpanContext._parentId) { + suiteTags[TEST_MODULE_ID] = testSuiteSpanContext._parentId.toString(10) + } + return suiteTags +} class MochaPlugin extends CiPlugin { static get id () { @@ -50,7 +73,8 @@ class MochaPlugin extends CiPlugin { if (!this.libraryConfig?.isCodeCoverageEnabled) { return } - const testSuiteSpan = this._testSuites.get(suiteFile) + const testSuite = getTestSuitePath(suiteFile, this.sourceRoot) + const testSuiteSpan = this._testSuites.get(testSuite) if (!coverageFiles.length) { this.telemetry.count(TELEMETRY_CODE_COVERAGE_EMPTY) @@ -73,16 +97,20 @@ class MochaPlugin extends CiPlugin { }) this.addSub('ci:mocha:test-suite:start', ({ - testSuite, + testSuiteAbsolutePath, isUnskippable, isForcedToRun, itrCorrelationId }) => { - const store = storage.getStore() + // If the test module span is undefined, the plugin has not been initialized correctly and we bail out + if (!this.testModuleSpan) { + return + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) const testSuiteMetadata = getTestSuiteCommonTags( this.command, this.frameworkVersion, - getTestSuitePath(testSuite, this.sourceRoot), + testSuite, 'mocha' ) if (isUnskippable) { @@ -109,6 +137,7 @@ class MochaPlugin extends CiPlugin { if (itrCorrelationId) { testSuiteSpan.setTag(ITR_CORRELATION_ID, itrCorrelationId) } + const store = storage.getStore() this.enter(testSuiteSpan, store) this._testSuites.set(testSuite, testSuiteSpan) }) @@ -142,6 +171,10 @@ class MochaPlugin extends CiPlugin { this.enter(span, store) }) + this.addSub('ci:mocha:worker:finish', () => { + this.tracer._exporter.flush() + }) + this.addSub('ci:mocha:test:finish', (status) => { const store = storage.getStore() const span = store?.span @@ -194,7 +227,8 @@ class MochaPlugin extends CiPlugin { hasForcedToRunSuites, hasUnskippableSuites, error, - isEarlyFlakeDetectionEnabled + isEarlyFlakeDetectionEnabled, + isParallel }) => { if (this.testSessionSpan) { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} @@ -206,6 +240,10 @@ class MochaPlugin extends CiPlugin { this.testModuleSpan.setTag('error', error) } + if (isParallel) { + this.testSessionSpan.setTag(MOCHA_IS_PARALLEL, 'true') + } + addIntelligentTestRunnerSpanTags( this.testSessionSpan, this.testModuleSpan, @@ -234,6 +272,37 @@ class MochaPlugin extends CiPlugin { this.libraryConfig = null this.tracer._exporter.flush() }) + + this.addSub('ci:mocha:worker-report:trace', (traces) => { + const formattedTraces = JSON.parse(traces).map(trace => + trace.map(span => { + const formattedSpan = { + ...span, + span_id: id(span.span_id), + trace_id: id(span.trace_id), + parent_id: id(span.parent_id) + } + if (formattedSpan.name === 'mocha.test') { + const testSuite = span.meta[TEST_SUITE] + const testSuiteSpan = this._testSuites.get(testSuite) + if (!testSuiteSpan) { + log.warn(`Test suite span not found for test span with test suite ${testSuite}`) + return formattedSpan + } + const suiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan) + formattedSpan.meta = { + ...formattedSpan.meta, + ...suiteTags + } + } + return formattedSpan + }) + ) + + formattedTraces.forEach(trace => { + this.tracer._exporter.export(trace) + }) + }) } startTestSpan (testInfo) { @@ -242,7 +311,8 @@ class MochaPlugin extends CiPlugin { title, isNew, isEfdRetry, - testStartLine + testStartLine, + isParallel } = testInfo const testName = removeEfdStringFromTestName(testInfo.testName) @@ -257,8 +327,12 @@ class MochaPlugin extends CiPlugin { extraTags[TEST_SOURCE_START] = testStartLine } + if (isParallel) { + extraTags[MOCHA_IS_PARALLEL] = 'true' + } + const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) - const testSuiteSpan = this._testSuites.get(testSuiteAbsolutePath) + const testSuiteSpan = this._testSuites.get(testSuite) if (this.repositoryRoot !== this.sourceRoot && !!this.repositoryRoot) { extraTags[TEST_SOURCE_FILE] = getTestSuitePath(testSuiteAbsolutePath, this.repositoryRoot) diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js index f91bdd52090..e74869dbe82 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/index.js @@ -4,7 +4,8 @@ const Writer = require('./writer') const { JEST_WORKER_COVERAGE_PAYLOAD_CODE, JEST_WORKER_TRACE_PAYLOAD_CODE, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE } = require('../../../plugins/util/test') function getInterprocessTraceCode () { @@ -14,6 +15,9 @@ function getInterprocessTraceCode () { if (process.env.CUCUMBER_WORKER_ID) { return CUCUMBER_WORKER_TRACE_PAYLOAD_CODE } + if (process.env.MOCHA_WORKER_ID) { + return MOCHA_WORKER_TRACE_PAYLOAD_CODE + } return null } diff --git a/packages/dd-trace/src/exporter.js b/packages/dd-trace/src/exporter.js index 01bb96ac380..02d50c3b57e 100644 --- a/packages/dd-trace/src/exporter.js +++ b/packages/dd-trace/src/exporter.js @@ -19,6 +19,7 @@ module.exports = name => { return require('./ci-visibility/exporters/agent-proxy') case exporters.JEST_WORKER: case exporters.CUCUMBER_WORKER: + case exporters.MOCHA_WORKER: return require('./ci-visibility/exporters/test-worker') default: return inAWSLambda && !usingLambdaExtension ? require('./exporters/log') : require('./exporters/agent') diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 82217ada440..d7193917b05 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -54,6 +54,7 @@ module.exports = { get 'microgateway-core' () { return require('../../../datadog-plugin-microgateway-core/src') }, get mocha () { return require('../../../datadog-plugin-mocha/src') }, get 'mocha-each' () { return require('../../../datadog-plugin-mocha/src') }, + get workerpool () { return require('../../../datadog-plugin-mocha/src') }, get moleculer () { return require('../../../datadog-plugin-moleculer/src') }, get mongodb () { return require('../../../datadog-plugin-mongodb-core/src') }, get 'mongodb-core' () { return require('../../../datadog-plugin-mongodb-core/src') }, diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 23ce067670a..d1d1861ea5d 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -62,6 +62,7 @@ const JEST_TEST_RUNNER = 'test.jest.test_runner' const JEST_DISPLAY_NAME = 'test.jest.display_name' const CUCUMBER_IS_PARALLEL = 'test.cucumber.is_parallel' +const MOCHA_IS_PARALLEL = 'test.mocha.is_parallel' const TEST_ITR_TESTS_SKIPPED = '_dd.ci.itr.tests_skipped' const TEST_ITR_SKIPPING_ENABLED = 'test.itr.tests_skipping.enabled' @@ -87,6 +88,9 @@ const JEST_WORKER_COVERAGE_PAYLOAD_CODE = 61 // cucumber worker variables const CUCUMBER_WORKER_TRACE_PAYLOAD_CODE = 70 +// mocha worker variables +const MOCHA_WORKER_TRACE_PAYLOAD_CODE = 80 + // Early flake detection util strings const EFD_STRING = "Retried by Datadog's Early Flake Detection" const EFD_TEST_NAME_REGEX = new RegExp(EFD_STRING + ' \\(#\\d+\\): ', 'g') @@ -98,6 +102,7 @@ module.exports = { JEST_TEST_RUNNER, JEST_DISPLAY_NAME, CUCUMBER_IS_PARALLEL, + MOCHA_IS_PARALLEL, TEST_TYPE, TEST_NAME, TEST_SUITE, @@ -111,6 +116,7 @@ module.exports = { JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE, TEST_SOURCE_START, TEST_SKIPPED_BY_ITR, TEST_CONFIGURATION_BROWSER_NAME, diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js index cb212eca891..3322fbb8e85 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -6,19 +6,23 @@ const TestWorkerCiVisibilityExporter = require('../../../../src/ci-visibility/ex const { JEST_WORKER_TRACE_PAYLOAD_CODE, JEST_WORKER_COVERAGE_PAYLOAD_CODE, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + MOCHA_WORKER_TRACE_PAYLOAD_CODE } = require('../../../../src/plugins/util/test') describe('CI Visibility Test Worker Exporter', () => { let send, originalSend + beforeEach(() => { send = sinon.spy() originalSend = process.send process.send = send }) + afterEach(() => { process.send = originalSend }) + context('when the process is a jest worker', () => { beforeEach(() => { process.env.JEST_WORKER_ID = '1' @@ -26,6 +30,7 @@ describe('CI Visibility Test Worker Exporter', () => { afterEach(() => { delete process.env.JEST_WORKER_ID }) + it('can export traces', () => { const trace = [{ type: 'test' }] const traceSecond = [{ type: 'test', name: 'other' }] @@ -35,6 +40,7 @@ describe('CI Visibility Test Worker Exporter', () => { jestWorkerExporter.flush() expect(send).to.have.been.calledWith([JEST_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) }) + it('can export coverages', () => { const coverage = { sessionId: '1', suiteId: '1', files: ['test.js'] } const coverageSecond = { sessionId: '2', suiteId: '2', files: ['test2.js'] } @@ -46,6 +52,7 @@ describe('CI Visibility Test Worker Exporter', () => { [JEST_WORKER_COVERAGE_PAYLOAD_CODE, JSON.stringify([coverage, coverageSecond])] ) }) + it('does not break if process.send is undefined', () => { delete process.send const trace = [{ type: 'test' }] @@ -55,6 +62,7 @@ describe('CI Visibility Test Worker Exporter', () => { expect(send).not.to.have.been.called }) }) + context('when the process is a cucumber worker', () => { beforeEach(() => { process.env.CUCUMBER_WORKER_ID = '1' @@ -62,6 +70,7 @@ describe('CI Visibility Test Worker Exporter', () => { afterEach(() => { delete process.env.CUCUMBER_WORKER_ID }) + it('can export traces', () => { const trace = [{ type: 'test' }] const traceSecond = [{ type: 'test', name: 'other' }] @@ -71,6 +80,7 @@ describe('CI Visibility Test Worker Exporter', () => { cucumberWorkerExporter.flush() expect(send).to.have.been.calledWith([CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) }) + it('does not break if process.send is undefined', () => { delete process.send const trace = [{ type: 'test' }] @@ -80,4 +90,32 @@ describe('CI Visibility Test Worker Exporter', () => { expect(send).not.to.have.been.called }) }) + + context('when the process is a mocha worker', () => { + beforeEach(() => { + process.env.MOCHA_WORKER_ID = '1' + }) + afterEach(() => { + delete process.env.MOCHA_WORKER_ID + }) + + it('can export traces', () => { + const trace = [{ type: 'test' }] + const traceSecond = [{ type: 'test', name: 'other' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.export(traceSecond) + mochaWorkerExporter.flush() + expect(send).to.have.been.calledWith([MOCHA_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace, traceSecond])]) + }) + + it('does not break if process.send is undefined', () => { + delete process.send + const trace = [{ type: 'test' }] + const mochaWorkerExporter = new TestWorkerCiVisibilityExporter() + mochaWorkerExporter.export(trace) + mochaWorkerExporter.flush() + expect(send).not.to.have.been.called + }) + }) })