diff --git a/integration-tests/ci-visibility/test-api-manual/run-fake-test-framework.js b/integration-tests/ci-visibility/test-api-manual/run-fake-test-framework.js new file mode 100644 index 00000000000..0916ddbd7dc --- /dev/null +++ b/integration-tests/ci-visibility/test-api-manual/run-fake-test-framework.js @@ -0,0 +1,27 @@ +'use strict' + +/* eslint-disable */ + +function runTests () { + const promises = global.tests.map(async (test) => { + let testStatus = 'pass' + let testError = null + global.beforeEachHooks.forEach(beforeEach => { + beforeEach(test.description) + }) + try { + await test.fn() + console.log(`✓ ${test.description}`) + } catch (e) { + testError = e + testStatus = 'fail' + console.log(`x ${test.description}: ${e}`) + } + global.afterEachHooks.forEach(afterEach => { + afterEach(testStatus, testError) + }) + }) + return Promise.all(promises) +} + +runTests() diff --git a/integration-tests/ci-visibility/test-api-manual/setup-fake-test-framework.js b/integration-tests/ci-visibility/test-api-manual/setup-fake-test-framework.js new file mode 100644 index 00000000000..af7b3ab16b3 --- /dev/null +++ b/integration-tests/ci-visibility/test-api-manual/setup-fake-test-framework.js @@ -0,0 +1,33 @@ +'use strict' + +global.tests = [] +global.beforeEachHooks = [] +global.afterEachHooks = [] + +function describe (description, cb) { + cb() +} + +function test (description, fn) { + global.tests.push({ description, fn }) +} + +function beforeEach (fn) { + global.beforeEachHooks.push(fn) +} + +function afterEach (fn) { + global.afterEachHooks.push(fn) +} + +global.describe = describe +global.test = test +global.beforeEach = beforeEach +global.afterEach = afterEach +global.assert = { + equal: (a, b) => { + if (a !== b) { + throw new Error(`${a} is not equal to ${b}`) + } + } +} diff --git a/integration-tests/ci-visibility/test-api-manual/test.fake.js b/integration-tests/ci-visibility/test-api-manual/test.fake.js new file mode 100644 index 00000000000..bc1f17972b7 --- /dev/null +++ b/integration-tests/ci-visibility/test-api-manual/test.fake.js @@ -0,0 +1,48 @@ +/* eslint-disable */ +const { channel } = require('diagnostics_channel') +const tracer = require('dd-trace') + +const testStartCh = channel('dd-trace:ci:manual:test:start') +const testFinishCh = channel('dd-trace:ci:manual:test:finish') +const testAddTagsCh = channel('dd-trace:ci:manual:test:addTags') +const testSuite = __filename + +describe('can run tests', () => { + beforeEach((testName) => { + testStartCh.publish({ testName, testSuite }) + }) + afterEach((status, error) => { + testFinishCh.publish({ status, error }) + }) + test('first test will pass', () => { + testAddTagsCh.publish({ 'test.custom.tag': 'custom.value' }) + assert.equal(1, 1) + }) + test('second test will fail', () => { + assert.equal(1, 2) + }) + test('async test will pass', () => { + return new Promise((resolve) => { + setTimeout(() => { + assert.equal(1, 1) + resolve() + }, 10) + }) + }) + test('integration test', () => { + // Just for testing purposes, so we don't create a custom span + if (!process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED) { + return Promise.resolve() + } + const testSpan = tracer.scope().active() + const childSpan = tracer.startSpan('custom.span', { + childOf: testSpan + }) + return new Promise((resolve) => { + setTimeout(() => { + childSpan.finish() + resolve() + }, 10) + }) + }) +}) diff --git a/integration-tests/test-api-manual.spec.js b/integration-tests/test-api-manual.spec.js new file mode 100644 index 00000000000..7873a300ac3 --- /dev/null +++ b/integration-tests/test-api-manual.spec.js @@ -0,0 +1,103 @@ +'use strict' + +const { exec } = require('child_process') + +const getPort = require('get-port') +const { assert } = require('chai') + +const { + createSandbox, + getCiVisAgentlessConfig +} = require('./helpers') +const { FakeCiVisIntake } = require('./ci-visibility-intake') +const webAppServer = require('./ci-visibility/web-app-server') +const { + TEST_STATUS +} = require('../packages/dd-trace/src/plugins/util/test') + +describe('test-api-manual', () => { + let sandbox, cwd, receiver, childProcess, webAppPort + before(async () => { + sandbox = await createSandbox([], true) + cwd = sandbox.folder + webAppPort = await getPort() + webAppServer.listen(webAppPort) + }) + + after(async () => { + await sandbox.remove() + await new Promise(resolve => webAppServer.close(resolve)) + }) + + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) + + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) + + it('can use the manual api', (done) => { + const receiverPromise = receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testEvents = events.filter(event => event.type === 'test') + assert.includeMembers(testEvents.map(test => test.content.resource), [ + 'ci-visibility/test-api-manual/test.fake.js.second test will fail', + 'ci-visibility/test-api-manual/test.fake.js.first test will pass', + 'ci-visibility/test-api-manual/test.fake.js.async test will pass', + 'ci-visibility/test-api-manual/test.fake.js.integration test' + ]) + + assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ + 'pass', + 'pass', + 'pass', + 'fail' + ]) + + const passedTest = testEvents.find( + test => test.content.resource === 'ci-visibility/test-api-manual/test.fake.js.first test will pass' + ) + assert.propertyVal(passedTest.content.meta, 'test.custom.tag', 'custom.value') + + const customSpan = events.find(event => event.type === 'span') + assert.propertyVal(customSpan.content, 'resource', 'custom.span') + }).catch(done) + + childProcess = exec( + 'node --require ./ci-visibility/test-api-manual/setup-fake-test-framework.js ' + + '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework', + { + cwd, + env: { ...getCiVisAgentlessConfig(receiver.port), DD_CIVISIBILITY_MANUAL_API_ENABLED: '1' }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + receiverPromise.then(() => done()) + }) + }) + + it('does not report test spans if DD_CIVISIBILITY_MANUAL_API_ENABLED is not set', (done) => { + receiver.assertPayloadReceived(() => { + const error = new Error('should not report spans') + done(error) + }, ({ url }) => url === '/api/v2/citestcycle').catch(() => {}) + + childProcess = exec( + 'node --require ./ci-visibility/test-api-manual/setup-fake-test-framework.js ' + + '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework', + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + done() + }) + }) +}) diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 85eade193cc..bbb3a9afe0a 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -70,7 +70,7 @@ class MochaPlugin extends CiPlugin { this.addSub('ci:mocha:test-suite:finish', (status) => { const store = storage.getStore() if (store && store.span) { - const span = storage.getStore().span + const span = store.span // the test status of the suite may have been set in ci:mocha:test-suite:error already if (!span.context()._tags[TEST_STATUS]) { span.setTag(TEST_STATUS, status) @@ -82,7 +82,7 @@ class MochaPlugin extends CiPlugin { this.addSub('ci:mocha:test-suite:error', (err) => { const store = storage.getStore() if (store && store.span) { - const span = storage.getStore().span + const span = store.span span.setTag('error', err) span.setTag(TEST_STATUS, 'fail') } @@ -99,7 +99,7 @@ class MochaPlugin extends CiPlugin { const store = storage.getStore() if (store && store.span) { - const span = storage.getStore().span + const span = store.span span.setTag(TEST_STATUS, status) diff --git a/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js b/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js new file mode 100644 index 00000000000..f6a1612b373 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js @@ -0,0 +1,45 @@ +const CiPlugin = require('../../plugins/ci_plugin') +const { + TEST_STATUS, + finishAllTraceSpans, + getTestSuitePath +} = require('../../plugins/util/test') +const { storage } = require('../../../../datadog-core') + +class TestApiManualPlugin extends CiPlugin { + static get id () { + return 'test-api-manual' + } + constructor (...args) { + super(...args) + this.sourceRoot = process.cwd() + + this.addSub('dd-trace:ci:manual:test:start', ({ testName, testSuite }) => { + const store = storage.getStore() + const testSuiteRelative = getTestSuitePath(testSuite, this.sourceRoot) + const testSpan = this.startTestSpan(testName, testSuiteRelative) + this.enter(testSpan, store) + }) + this.addSub('dd-trace:ci:manual:test:finish', ({ status, error }) => { + const store = storage.getStore() + const testSpan = store && store.span + if (testSpan) { + testSpan.setTag(TEST_STATUS, status) + if (error) { + testSpan.setTag('error', error) + } + testSpan.finish() + finishAllTraceSpans(testSpan) + } + }) + this.addSub('dd-trace:ci:manual:test:addTags', (tags) => { + const store = storage.getStore() + const testSpan = store && store.span + if (testSpan) { + testSpan.addTags(tags) + } + }) + } +} + +module.exports = TestApiManualPlugin diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 661b9bcff6b..ca14061b252 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -172,6 +172,11 @@ class Config { true ) + const DD_CIVISIBILITY_MANUAL_API_ENABLED = coalesce( + process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED, + false + ) + const DD_SERVICE = options.service || process.env.DD_SERVICE || process.env.DD_SERVICE_NAME || @@ -575,6 +580,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?) (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED)) this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED) + this.isManualApiEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_MANUAL_API_ENABLED) this.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT diff --git a/packages/dd-trace/src/plugin_manager.js b/packages/dd-trace/src/plugin_manager.js index fe500edde82..1f705f4c553 100644 --- a/packages/dd-trace/src/plugin_manager.js +++ b/packages/dd-trace/src/plugin_manager.js @@ -21,7 +21,7 @@ const disabledPlugins = new Set( DD_TRACE_DISABLED_PLUGINS && DD_TRACE_DISABLED_PLUGINS.split(',').map(plugin => plugin.trim()) ) -// TODO actually ... should we be looking at envrionment variables this deep down in the code? +// TODO actually ... should we be looking at environment variables this deep down in the code? const pluginClasses = {} diff --git a/packages/dd-trace/src/proxy.js b/packages/dd-trace/src/proxy.js index b9a18744672..b8d4c9904e5 100644 --- a/packages/dd-trace/src/proxy.js +++ b/packages/dd-trace/src/proxy.js @@ -64,6 +64,12 @@ class Tracer extends NoopProxy { this._pluginManager.configure(config) setStartupLogPluginManager(this._pluginManager) telemetry.start(config, this._pluginManager) + + if (config.isManualApiEnabled) { + const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin') + this._testApiManualPlugin = new TestApiManualPlugin(this) + this._testApiManualPlugin.configure({ ...config, enabled: true }) + } } } catch (e) { log.error(e) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index f3760306e5f..af4a66cb9bc 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -1059,6 +1059,7 @@ describe('Config', () => { beforeEach(() => { delete process.env.DD_CIVISIBILITY_ITR_ENABLED delete process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED + delete process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED options = {} }) context('ci visibility mode is enabled', () => { @@ -1083,6 +1084,15 @@ describe('Config', () => { const config = new Config(options) expect(config).to.have.property('isIntelligentTestRunnerEnabled', false) }) + it('should disable manual testing API by default', () => { + const config = new Config(options) + expect(config).to.have.property('isManualApiEnabled', false) + }) + it('should enable manual testing API if DD_CIVISIBILITY_MANUAL_API_ENABLED is passed', () => { + process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED = 'true' + const config = new Config(options) + expect(config).to.have.property('isManualApiEnabled', true) + }) }) context('ci visibility mode is not enabled', () => { it('should not activate intelligent test runner or git metadata upload', () => {