[ci-visibility] Support CI Visibility's manual API (#3202)

integration-tests/ci-visibility/test-api-manual/run-fake-test-framework.js

@@ -0,0 +1,27 @@
+'use strict'
+
+/* eslint-disable */
+
+function runTests () {
+  const promises = global.tests.map(async (test) => {
+    let testStatus = 'pass'
+    let testError = null
+    global.beforeEachHooks.forEach(beforeEach => {
+      beforeEach(test.description)
+    })
+    try {
+      await test.fn()
+      console.log(`✓ ${test.description}`)
+    } catch (e) {
+      testError = e
+      testStatus = 'fail'
+      console.log(`x ${test.description}: ${e}`)
+    }
+    global.afterEachHooks.forEach(afterEach => {
+      afterEach(testStatus, testError)
+    })
+  })
+  return Promise.all(promises)
+}
+
+runTests()

integration-tests/ci-visibility/test-api-manual/setup-fake-test-framework.js

@@ -0,0 +1,33 @@
+'use strict'
+
+global.tests = []
+global.beforeEachHooks = []
+global.afterEachHooks = []
+
+function describe (description, cb) {
+  cb()
+}
+
+function test (description, fn) {
+  global.tests.push({ description, fn })
+}
+
+function beforeEach (fn) {
+  global.beforeEachHooks.push(fn)
+}
+
+function afterEach (fn) {
+  global.afterEachHooks.push(fn)
+}
+
+global.describe = describe
+global.test = test
+global.beforeEach = beforeEach
+global.afterEach = afterEach
+global.assert = {
+  equal: (a, b) => {
+    if (a !== b) {
+      throw new Error(`${a} is not equal to ${b}`)
+    }
+  }
+}

integration-tests/ci-visibility/test-api-manual/test.fake.js

@@ -0,0 +1,48 @@
+/* eslint-disable */
+const { channel } = require('diagnostics_channel')
+const tracer = require('dd-trace')
+
+const testStartCh = channel('dd-trace:ci:manual:test:start')
+const testFinishCh = channel('dd-trace:ci:manual:test:finish')
+const testAddTagsCh = channel('dd-trace:ci:manual:test:addTags')
+const testSuite = __filename
+
+describe('can run tests', () => {
+  beforeEach((testName) => {
+    testStartCh.publish({ testName, testSuite })
+  })
+  afterEach((status, error) => {
+    testFinishCh.publish({ status, error })
+  })
+  test('first test will pass', () => {
+    testAddTagsCh.publish({ 'test.custom.tag': 'custom.value' })
+    assert.equal(1, 1)
+  })
+  test('second test will fail', () => {
+    assert.equal(1, 2)
+  })
+  test('async test will pass', () => {
+    return new Promise((resolve) => {
+      setTimeout(() => {
+        assert.equal(1, 1)
+        resolve()
+      }, 10)
+    })
+  })
+  test('integration test', () => {
+    // Just for testing purposes, so we don't create a custom span
+    if (!process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED) {
+      return Promise.resolve()
+    }
+    const testSpan = tracer.scope().active()
+    const childSpan = tracer.startSpan('custom.span', {
+      childOf: testSpan
+    })
+    return new Promise((resolve) => {
+      setTimeout(() => {
+        childSpan.finish()
+        resolve()
+      }, 10)
+    })
+  })
+})

integration-tests/test-api-manual.spec.js

@@ -0,0 +1,103 @@
+'use strict'
+
+const { exec } = require('child_process')
+
+const getPort = require('get-port')
+const { assert } = require('chai')
+
+const {
+  createSandbox,
+  getCiVisAgentlessConfig
+} = require('./helpers')
+const { FakeCiVisIntake } = require('./ci-visibility-intake')
+const webAppServer = require('./ci-visibility/web-app-server')
+const {
+  TEST_STATUS
+} = require('../packages/dd-trace/src/plugins/util/test')
+
+describe('test-api-manual', () => {
+  let sandbox, cwd, receiver, childProcess, webAppPort
+  before(async () => {
+    sandbox = await createSandbox([], true)
+    cwd = sandbox.folder
+    webAppPort = await getPort()
+    webAppServer.listen(webAppPort)
+  })
+
+  after(async () => {
+    await sandbox.remove()
+    await new Promise(resolve => webAppServer.close(resolve))
+  })
+
+  beforeEach(async function () {
+    const port = await getPort()
+    receiver = await new FakeCiVisIntake(port).start()
+  })
+
+  afterEach(async () => {
+    childProcess.kill()
+    await receiver.stop()
+  })
+
+  it('can use the manual api', (done) => {
+    const receiverPromise = receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcycle', payloads => {
+      const events = payloads.flatMap(({ payload }) => payload.events)
+
+      const testEvents = events.filter(event => event.type === 'test')
+      assert.includeMembers(testEvents.map(test => test.content.resource), [
+        'ci-visibility/test-api-manual/test.fake.js.second test will fail',
+        'ci-visibility/test-api-manual/test.fake.js.first test will pass',
+        'ci-visibility/test-api-manual/test.fake.js.async test will pass',
+        'ci-visibility/test-api-manual/test.fake.js.integration test'
+      ])
+
+      assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [
+        'pass',
+        'pass',
+        'pass',
+        'fail'
+      ])
+
+      const passedTest = testEvents.find(
+        test => test.content.resource === 'ci-visibility/test-api-manual/test.fake.js.first test will pass'
+      )
+      assert.propertyVal(passedTest.content.meta, 'test.custom.tag', 'custom.value')
+
+      const customSpan = events.find(event => event.type === 'span')
+      assert.propertyVal(customSpan.content, 'resource', 'custom.span')
+    }).catch(done)
+
+    childProcess = exec(
+      'node --require ./ci-visibility/test-api-manual/setup-fake-test-framework.js ' +
+      '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework',
+      {
+        cwd,
+        env: { ...getCiVisAgentlessConfig(receiver.port), DD_CIVISIBILITY_MANUAL_API_ENABLED: '1' },
+        stdio: 'pipe'
+      }
+    )
+    childProcess.on('exit', () => {
+      receiverPromise.then(() => done())
+    })
+  })
+
+  it('does not report test spans if DD_CIVISIBILITY_MANUAL_API_ENABLED is not set', (done) => {
+    receiver.assertPayloadReceived(() => {
+      const error = new Error('should not report spans')
+      done(error)
+    }, ({ url }) => url === '/api/v2/citestcycle').catch(() => {})
+
+    childProcess = exec(
+      'node --require ./ci-visibility/test-api-manual/setup-fake-test-framework.js ' +
+      '--require ./ci-visibility/test-api-manual/test.fake.js ./ci-visibility/test-api-manual/run-fake-test-framework',
+      {
+        cwd,
+        env: getCiVisAgentlessConfig(receiver.port),
+        stdio: 'pipe'
+      }
+    )
+    childProcess.on('exit', () => {
+      done()
+    })
+  })
+})

packages/datadog-plugin-mocha/src/index.js

@@ -70,7 +70,7 @@ class MochaPlugin extends CiPlugin {
     this.addSub('ci:mocha:test-suite:finish', (status) => {
       const store = storage.getStore()
       if (store && store.span) {
-        const span = storage.getStore().span
+        const span = store.span
         // the test status of the suite may have been set in ci:mocha:test-suite:error already
         if (!span.context()._tags[TEST_STATUS]) {
           span.setTag(TEST_STATUS, status)
@@ -82,7 +82,7 @@ class MochaPlugin extends CiPlugin {
     this.addSub('ci:mocha:test-suite:error', (err) => {
       const store = storage.getStore()
       if (store && store.span) {
-        const span = storage.getStore().span
+        const span = store.span
         span.setTag('error', err)
         span.setTag(TEST_STATUS, 'fail')
       }
@@ -99,7 +99,7 @@ class MochaPlugin extends CiPlugin {
       const store = storage.getStore()
 
       if (store && store.span) {
-        const span = storage.getStore().span
+        const span = store.span
 
         span.setTag(TEST_STATUS, status)
 

packages/dd-trace/src/ci-visibility/test-api-manual/test-api-manual-plugin.js

@@ -0,0 +1,45 @@
+const CiPlugin = require('../../plugins/ci_plugin')
+const {
+  TEST_STATUS,
+  finishAllTraceSpans,
+  getTestSuitePath
+} = require('../../plugins/util/test')
+const { storage } = require('../../../../datadog-core')
+
+class TestApiManualPlugin extends CiPlugin {
+  static get id () {
+    return 'test-api-manual'
+  }
+  constructor (...args) {
+    super(...args)
+    this.sourceRoot = process.cwd()
+
+    this.addSub('dd-trace:ci:manual:test:start', ({ testName, testSuite }) => {
+      const store = storage.getStore()
+      const testSuiteRelative = getTestSuitePath(testSuite, this.sourceRoot)
+      const testSpan = this.startTestSpan(testName, testSuiteRelative)
+      this.enter(testSpan, store)
+    })
+    this.addSub('dd-trace:ci:manual:test:finish', ({ status, error }) => {
+      const store = storage.getStore()
+      const testSpan = store && store.span
+      if (testSpan) {
+        testSpan.setTag(TEST_STATUS, status)
+        if (error) {
+          testSpan.setTag('error', error)
+        }
+        testSpan.finish()
+        finishAllTraceSpans(testSpan)
+      }
+    })
+    this.addSub('dd-trace:ci:manual:test:addTags', (tags) => {
+      const store = storage.getStore()
+      const testSpan = store && store.span
+      if (testSpan) {
+        testSpan.addTags(tags)
+      }
+    })
+  }
+}
+
+module.exports = TestApiManualPlugin

packages/dd-trace/src/config.js

@@ -169,6 +169,11 @@ class Config {
       true
     )
 
+    const DD_CIVISIBILITY_MANUAL_API_ENABLED = coalesce(
+      process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED,
+      false
+    )
+
     const DD_SERVICE = options.service ||
       process.env.DD_SERVICE ||
       process.env.DD_SERVICE_NAME ||
@@ -572,6 +577,7 @@ ken|consumer_?(?:id|key|secret)|sign(?:ed|ature)?|auth(?:entication|orization)?)
       (this.isIntelligentTestRunnerEnabled && !isFalse(DD_CIVISIBILITY_GIT_UPLOAD_ENABLED))
 
     this.gitMetadataEnabled = isTrue(DD_TRACE_GIT_METADATA_ENABLED)
+    this.isManualApiEnabled = this.isCiVisibility && isTrue(DD_CIVISIBILITY_MANUAL_API_ENABLED)
 
     this.openaiSpanCharLimit = DD_OPENAI_SPAN_CHAR_LIMIT
 

packages/dd-trace/src/plugin_manager.js

@@ -21,7 +21,7 @@ const disabledPlugins = new Set(
   DD_TRACE_DISABLED_PLUGINS && DD_TRACE_DISABLED_PLUGINS.split(',').map(plugin => plugin.trim())
 )
 
-// TODO actually ... should we be looking at envrionment variables this deep down in the code?
+// TODO actually ... should we be looking at environment variables this deep down in the code?
 
 const pluginClasses = {}
 

packages/dd-trace/src/proxy.js

@@ -64,6 +64,12 @@ class Tracer extends NoopProxy {
         this._pluginManager.configure(config)
         setStartupLogPluginManager(this._pluginManager)
         telemetry.start(config, this._pluginManager)
+
+        if (config.isManualApiEnabled) {
+          const TestApiManualPlugin = require('./ci-visibility/test-api-manual/test-api-manual-plugin')
+          this._testApiManualPlugin = new TestApiManualPlugin(this)
+          this._testApiManualPlugin.configure({ ...config, enabled: true })
+        }
       }
     } catch (e) {
       log.error(e)

packages/dd-trace/test/config.spec.js

@@ -1059,6 +1059,7 @@ describe('Config', () => {
     beforeEach(() => {
       delete process.env.DD_CIVISIBILITY_ITR_ENABLED
       delete process.env.DD_CIVISIBILITY_GIT_UPLOAD_ENABLED
+      delete process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED
       options = {}
     })
     context('ci visibility mode is enabled', () => {
@@ -1083,6 +1084,15 @@ describe('Config', () => {
         const config = new Config(options)
         expect(config).to.have.property('isIntelligentTestRunnerEnabled', false)
       })
+      it('should disable manual testing API by default', () => {
+        const config = new Config(options)
+        expect(config).to.have.property('isManualApiEnabled', false)
+      })
+      it('should enable manual testing API if DD_CIVISIBILITY_MANUAL_API_ENABLED is passed', () => {
+        process.env.DD_CIVISIBILITY_MANUAL_API_ENABLED = 'true'
+        const config = new Config(options)
+        expect(config).to.have.property('isManualApiEnabled', true)
+      })
     })
     context('ci visibility mode is not enabled', () => {
       it('should not activate intelligent test runner or git metadata upload', () => {