From ae76c38146777f23c274da323d6163c277edcc4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 30 Sep 2024 11:36:07 +0200 Subject: [PATCH 01/27] [test visibility] Support early flake detection for cucumber (#4733) --- integration-tests/cucumber/cucumber.spec.js | 2325 ++++++++++------- .../datadog-instrumentations/src/cucumber.js | 112 +- packages/datadog-plugin-cucumber/src/index.js | 5 + 3 files changed, 1410 insertions(+), 1032 deletions(-) diff --git a/integration-tests/cucumber/cucumber.spec.js b/integration-tests/cucumber/cucumber.spec.js index bcf768883e8..35c4b3b2060 100644 --- a/integration-tests/cucumber/cucumber.spec.js +++ b/integration-tests/cucumber/cucumber.spec.js @@ -29,6 +29,7 @@ const { TEST_SOURCE_FILE, TEST_SOURCE_START, TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, TEST_IS_NEW, TEST_IS_RETRY, TEST_NAME, @@ -43,357 +44,415 @@ const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/ const isOldNode = semver.satisfies(process.version, '<=16') const versions = ['7.0.0', isOldNode ? '9' : 'latest'] -const moduleType = [ - { - type: 'commonJS', - runTestsCommand: './node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - runTestsWithCoverageCommand: - './node_modules/nyc/bin/nyc.js -r=text-summary ' + - 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - parallelModeCommand: './node_modules/.bin/cucumber-js ' + - 'ci-visibility/features/*.feature --parallel 2', - featuresPath: 'ci-visibility/features/', - fileExtension: 'js' - } -] +const runTestsCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const runTestsWithCoverageCommand = './node_modules/nyc/bin/nyc.js -r=text-summary ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature' +const parallelModeCommand = './node_modules/.bin/cucumber-js ci-visibility/features/*.feature --parallel 2' +const featuresPath = 'ci-visibility/features/' +const fileExtension = 'js' versions.forEach(version => { - moduleType.forEach(({ - type, - runTestsCommand, - runTestsWithCoverageCommand, - parallelModeCommand, - featuresPath, - fileExtension - }) => { - // TODO: add esm tests - describe(`cucumber@${version} ${type}`, () => { - let sandbox, cwd, receiver, childProcess, testOutput - - before(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) - - sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', 'nyc'], true) - cwd = sandbox.folder - }) + // TODO: add esm tests + describe(`cucumber@${version} commonJS`, () => { + let sandbox, cwd, receiver, childProcess, testOutput - after(async function () { - // add an explicit timeout to make tests less flaky - this.timeout(50000) + before(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - await sandbox.remove() - }) + sandbox = await createSandbox([`@cucumber/cucumber@${version}`, 'assert', 'nyc'], true) + cwd = sandbox.folder + }) - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) + after(async function () { + // add an explicit timeout to make tests less flaky + this.timeout(50000) - afterEach(async () => { - testOutput = '' - childProcess.kill() - await receiver.stop() - }) + await sandbox.remove() + }) - const reportMethods = ['agentless', 'evp proxy'] + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) - reportMethods.forEach((reportMethod) => { - context(`reporting via ${reportMethod}`, () => { - let envVars, isAgentless - beforeEach(() => { - isAgentless = reportMethod === 'agentless' - envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) - }) - const runModes = ['serial'] + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) - if (version !== '7.0.0') { // only on latest or 9 if node is old - runModes.push('parallel') - } + const reportMethods = ['agentless', 'evp proxy'] - runModes.forEach((runMode) => { - it(`(${runMode}) can run and report tests`, (done) => { - const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand + reportMethods.forEach((reportMethod) => { + context(`reporting via ${reportMethod}`, () => { + let envVars, isAgentless + beforeEach(() => { + isAgentless = reportMethod === 'agentless' + envVars = isAgentless ? getCiVisAgentlessConfig(receiver.port) : getCiVisEvpProxyConfig(receiver.port) + }) + const runModes = ['serial'] + + if (version !== '7.0.0') { // only on latest or 9 if node is old + runModes.push('parallel') + } + + runModes.forEach((runMode) => { + it(`(${runMode}) can run and report tests`, (done) => { + const runCommand = runMode === 'parallel' ? parallelModeCommand : runTestsCommand + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) + metadataDicts.forEach(metadata => { + for (const testLevel of TEST_LEVEL_EVENT_TYPES) { + assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') + } + }) - const receiverPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const metadataDicts = payloads.flatMap(({ payload }) => payload.metadata) - metadataDicts.forEach(metadata => { - for (const testLevel of TEST_LEVEL_EVENT_TYPES) { - assert.equal(metadata[testLevel][TEST_SESSION_NAME], 'my-test-session') - } - }) + const events = payloads.flatMap(({ payload }) => payload.events) - const events = payloads.flatMap(({ payload }) => payload.events) + const testSessionEvent = events.find(event => event.type === 'test_session_end') + const testModuleEvent = events.find(event => event.type === 'test_module_end') + const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') + const testEvents = events.filter(event => event.type === 'test') - const testSessionEvent = events.find(event => event.type === 'test_session_end') - const testModuleEvent = events.find(event => event.type === 'test_module_end') - const testSuiteEvents = events.filter(event => event.type === 'test_suite_end') - const testEvents = events.filter(event => event.type === 'test') + const stepEvents = events.filter(event => event.type === 'span') - const stepEvents = events.filter(event => event.type === 'span') + const { content: testSessionEventContent } = testSessionEvent + const { content: testModuleEventContent } = testModuleEvent - const { content: testSessionEventContent } = testSessionEvent - const { content: testModuleEventContent } = testModuleEvent + if (runMode === 'parallel') { + assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') + } - if (runMode === 'parallel') { - assert.equal(testSessionEventContent.meta[CUCUMBER_IS_PARALLEL], 'true') - } + assert.exists(testSessionEventContent.test_session_id) + assert.exists(testSessionEventContent.meta[TEST_COMMAND]) + assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) + assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) + assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') + + assert.exists(testModuleEventContent.test_session_id) + assert.exists(testModuleEventContent.test_module_id) + assert.exists(testModuleEventContent.meta[TEST_COMMAND]) + assert.exists(testModuleEventContent.meta[TEST_MODULE]) + assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) + assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') + assert.equal( + testModuleEventContent.test_session_id.toString(10), + testSessionEventContent.test_session_id.toString(10) + ) - assert.exists(testSessionEventContent.test_session_id) - assert.exists(testSessionEventContent.meta[TEST_COMMAND]) - assert.exists(testSessionEventContent.meta[TEST_TOOLCHAIN]) - assert.equal(testSessionEventContent.resource.startsWith('test_session.'), true) - assert.equal(testSessionEventContent.meta[TEST_STATUS], 'fail') - - assert.exists(testModuleEventContent.test_session_id) - assert.exists(testModuleEventContent.test_module_id) - assert.exists(testModuleEventContent.meta[TEST_COMMAND]) - assert.exists(testModuleEventContent.meta[TEST_MODULE]) - assert.equal(testModuleEventContent.resource.startsWith('test_module.'), true) - assert.equal(testModuleEventContent.meta[TEST_STATUS], 'fail') - assert.equal( - testModuleEventContent.test_session_id.toString(10), - testSessionEventContent.test_session_id.toString(10) - ) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ + `test_suite.${featuresPath}farewell.feature`, + `test_suite.${featuresPath}greetings.feature` + ]) + assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ + 'pass', + 'fail' + ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.resource), [ - `test_suite.${featuresPath}farewell.feature`, - `test_suite.${featuresPath}greetings.feature` - ]) - assert.includeMembers(testSuiteEvents.map(suite => suite.content.meta[TEST_STATUS]), [ - 'pass', - 'fail' - ]) - - testSuiteEvents.forEach(({ - content: { - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath)) - assert.equal(metrics[TEST_SOURCE_START], 1) - assert.exists(metrics[DD_HOST_CPU_COUNT]) - }) + testSuiteEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.isTrue(meta[TEST_SOURCE_FILE].startsWith(featuresPath)) + assert.equal(metrics[TEST_SOURCE_START], 1) + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) - assert.includeMembers(testEvents.map(test => test.content.resource), [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ]) - assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ - 'pass', - 'pass', - 'pass', - 'fail', - 'skip' - ]) - - testEvents.forEach(({ - content: { - meta, - metrics, - test_suite_id: testSuiteId, - test_module_id: testModuleId, - test_session_id: testSessionId - } - }) => { - assert.exists(meta[TEST_COMMAND]) - assert.exists(meta[TEST_MODULE]) - assert.exists(testSuiteId) - assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) - assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) - assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) - // Can read DD_TAGS - assert.propertyVal(meta, 'test.customtag', 'customvalue') - assert.propertyVal(meta, 'test.customtag2', 'customvalue2') - if (runMode === 'parallel') { - assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') - } - assert.exists(metrics[DD_HOST_CPU_COUNT]) - }) + assert.includeMembers(testEvents.map(test => test.content.resource), [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ]) + assert.includeMembers(testEvents.map(test => test.content.meta[TEST_STATUS]), [ + 'pass', + 'pass', + 'pass', + 'fail', + 'skip' + ]) - stepEvents.forEach(stepEvent => { - assert.equal(stepEvent.content.name, 'cucumber.step') - assert.property(stepEvent.content.meta, 'cucumber.step') - }) - }, 5000) + testEvents.forEach(({ + content: { + meta, + metrics, + test_suite_id: testSuiteId, + test_module_id: testModuleId, + test_session_id: testSessionId + } + }) => { + assert.exists(meta[TEST_COMMAND]) + assert.exists(meta[TEST_MODULE]) + assert.exists(testSuiteId) + assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) + assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) + assert.equal(meta[TEST_SOURCE_FILE].startsWith('ci-visibility/features'), true) + // Can read DD_TAGS + assert.propertyVal(meta, 'test.customtag', 'customvalue') + assert.propertyVal(meta, 'test.customtag2', 'customvalue2') + if (runMode === 'parallel') { + assert.propertyVal(meta, CUCUMBER_IS_PARALLEL, 'true') + } + assert.exists(metrics[DD_HOST_CPU_COUNT]) + }) - childProcess = exec( - runCommand, - { - cwd, - env: { - ...envVars, - DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', - DD_TEST_SESSION_NAME: 'my-test-session' - }, - stdio: 'pipe' - } - ) + stepEvents.forEach(stepEvent => { + assert.equal(stepEvent.content.name, 'cucumber.step') + assert.property(stepEvent.content.meta, 'cucumber.step') + }) + }, 5000) + + childProcess = exec( + runCommand, + { + cwd, + env: { + ...envVars, + DD_TAGS: 'test.customtag:customvalue,test.customtag2:customvalue2', + DD_TEST_SESSION_NAME: 'my-test-session' + }, + stdio: 'pipe' + } + ) - childProcess.on('exit', () => { - receiverPromise.then(() => done()).catch(done) - }) + childProcess.on('exit', () => { + receiverPromise.then(() => done()).catch(done) }) }) - context('intelligent test runner', () => { - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + }) + context('intelligent test runner', () => { + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + ) + const packfileRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise, + eventsRequestPromise + ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { + if (isAgentless) { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(searchCommitRequest.headers, 'dd-api-key') + assert.notProperty(packfileRequest.headers, 'dd-api-key') + } + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) - const packfileRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + assert.equal(numSuites, 2) - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise, - eventsRequestPromise - ]).then(([searchCommitRequest, packfileRequest, eventsRequest]) => { - if (isAgentless) { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(searchCommitRequest.headers, 'dd-api-key') - assert.notProperty(packfileRequest.headers, 'dd-api-key') - } + done() + }).catch(done) - const eventTypes = eventsRequest.payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + }) + it('can report code coverage', (done) => { + const libraryConfigRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') + ) + const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) + const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) + + Promise.all([ + libraryConfigRequestPromise, + codeCovRequestPromise, + eventsRequestPromise + ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { + const [coveragePayload] = codeCovRequest.payload + if (isAgentless) { + assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') + assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + } else { + assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') + assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + } + + assert.propertyVal(coveragePayload, 'name', 'coverage1') + assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') + assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + assert.include(coveragePayload.content, { + version: 2 + }) + const allCoverageFiles = codeCovRequest.payload + .flatMap(coverage => coverage.content.coverages) + .flatMap(file => file.files) + .map(file => file.filename) + + assert.includeMembers(allCoverageFiles, [ + `${featuresPath}support/steps.${fileExtension}`, + `${featuresPath}farewell.feature`, + `${featuresPath}greetings.feature` + ]) + // steps is twice because there are two suites using it + assert.equal( + allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, + 2 + ) + assert.exists(coveragePayload.content.coverages[0].test_session_id) + assert.exists(coveragePayload.content.coverages[0].test_suite_id) + + const testSession = eventsRequest + .payload + .events + .find(event => event.type === 'test_session_end') + .content + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + + const eventTypes = eventsRequest.payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }).catch(done) - done() - }).catch(done) + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.on('exit', () => { + // check that reported coverage is still the same + assert.include(testOutput, 'Lines : 100%') + done() + }) + }) + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false + }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', + (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` } - ) - }) - it('can report code coverage', (done) => { - const libraryConfigRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/libraries/tests/services/setting') - ) - const codeCovRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) + }]) + + const skippableRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) Promise.all([ - libraryConfigRequestPromise, - codeCovRequestPromise, + skippableRequestPromise, + coverageRequestPromise, eventsRequestPromise - ]).then(([libraryConfigRequest, codeCovRequest, eventsRequest]) => { - const [coveragePayload] = codeCovRequest.payload + ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { + const [coveragePayload] = coverageRequest.payload if (isAgentless) { - assert.propertyVal(libraryConfigRequest.headers, 'dd-api-key', '1') - assert.propertyVal(codeCovRequest.headers, 'dd-api-key', '1') + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') + assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') } else { - assert.notProperty(libraryConfigRequest.headers, 'dd-api-key') - assert.notProperty(codeCovRequest.headers, 'dd-api-key', '1') + assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') + assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') + assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') } - assert.propertyVal(coveragePayload, 'name', 'coverage1') assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') assert.propertyVal(coveragePayload, 'type', 'application/msgpack') - assert.include(coveragePayload.content, { - version: 2 - }) - const allCoverageFiles = codeCovRequest.payload - .flatMap(coverage => coverage.content.coverages) - .flatMap(file => file.files) - .map(file => file.filename) - - assert.includeMembers(allCoverageFiles, [ - `${featuresPath}support/steps.${fileExtension}`, - `${featuresPath}farewell.feature`, - `${featuresPath}greetings.feature` - ]) - // steps is twice because there are two suites using it - assert.equal( - allCoverageFiles.filter(file => file === `${featuresPath}support/steps.${fileExtension}`).length, - 2 - ) - assert.exists(coveragePayload.content.coverages[0].test_session_id) - assert.exists(coveragePayload.content.coverages[0].test_suite_id) - - const testSession = eventsRequest - .payload - .events - .find(event => event.type === 'test_session_end') - .content - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) const eventTypes = eventsRequest.payload.events.map(event => event.type) + + const skippedSuite = eventsRequest.payload.events.find(event => + event.content.resource === `test_suite.${featuresPath}farewell.feature` + ).content + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) const numSuites = eventTypes.reduce( (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 ) assert.equal(numSuites, 2) - }).catch(done) + const testSession = eventsRequest + .payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.on('exit', () => { - // check that reported coverage is still the same - assert.include(testOutput, 'Lines : 100%') + const testModule = eventsRequest + .payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) done() - }) - }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false - }) - - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) - - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - const testSession = payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - assert.exists(testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT]) - const testModule = payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'false') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'false') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + }).catch(done) childProcess = exec( runTestsWithCoverageCommand, @@ -404,692 +463,763 @@ versions.forEach(version => { } ) }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', - (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) - - const skippableRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) - const coverageRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcov')) - const eventsRequestPromise = receiver.payloadReceived(({ url }) => url.endsWith('/api/v2/citestcycle')) - - Promise.all([ - skippableRequestPromise, - coverageRequestPromise, - eventsRequestPromise - ]).then(([skippableRequest, coverageRequest, eventsRequest]) => { - const [coveragePayload] = coverageRequest.payload - if (isAgentless) { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - assert.propertyVal(coverageRequest.headers, 'dd-api-key', '1') - assert.propertyVal(eventsRequest.headers, 'dd-api-key', '1') - } else { - assert.notProperty(skippableRequest.headers, 'dd-api-key', '1') - assert.notProperty(coverageRequest.headers, 'dd-api-key', '1') - assert.notProperty(eventsRequest.headers, 'dd-api-key', '1') - } - assert.propertyVal(coveragePayload, 'name', 'coverage1') - assert.propertyVal(coveragePayload, 'filename', 'coverage1.msgpack') - assert.propertyVal(coveragePayload, 'type', 'application/msgpack') + it('does not skip tests if git metadata upload fails', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.setGitUploadStatus(404) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + const testSession = payload.events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + }) + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: false + }) - const eventTypes = eventsRequest.payload.events.map(event => event.type) + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }]) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + + receiver.assertPayloadReceived(({ payload }) => { + const eventTypes = payload.events.map(event => event.type) + // because they are not skipped + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + const numSuites = eventTypes.reduce( + (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + ) + assert.equal(numSuites, 2) + }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisAgentlessConfig(receiver.port), + stdio: 'inherit' + } + ) + }) + it('does not skip suites if suite is marked as unskippable', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) - const skippedSuite = eventsRequest.payload.events.find(event => - event.content.resource === `test_suite.${featuresPath}farewell.feature` - ).content - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.propertyVal(skippedSuite.meta, TEST_SKIPPED_BY_ITR, 'true') + receiver.setSuitesToSkip([ + { + type: 'suite', + attributes: { + suite: `${featuresPath}farewell.feature` + } + }, + { + type: 'suite', + attributes: { + suite: `${featuresPath}greetings.feature` + } + } + ]) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') + + assert.equal(suites.length, 2) + + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' + ).content + const forcedToRunSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ).content + + assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 - ) - assert.equal(numSuites, 2) - const testSession = eventsRequest - .payload.events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 1) - - const testModule = eventsRequest - .payload.events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_TYPE, 'suite') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 1) - done() - }).catch(done) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('only sets forced to run if suite was going to be skipped by ITR', (done) => { + receiver.setSettings({ + itr_enabled: true, + code_coverage: true, + tests_skipping: true + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - }) - it('does not skip tests if git metadata upload fails', (done) => { - receiver.setSuitesToSkip([{ + receiver.setSuitesToSkip([ + { type: 'suite', attributes: { suite: `${featuresPath}farewell.feature` } - }]) + } + ]) - receiver.setGitUploadStatus(404) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const suites = events.filter(event => event.type === 'test_suite_end') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + assert.equal(suites.length, 2) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + const testSession = events.find(event => event.type === 'test_session_end').content + const testModule = events.find(event => event.type === 'test_session_end').content + + assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) + assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) + + const skippedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' ) - assert.equal(numSuites, 2) - const testSession = payload.events.find(event => event.type === 'test_session_end').content + const failedSuite = suites.find( + event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' + ) + + assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') + assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) + assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + + assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') + assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') + assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { + receiver.setSuitesToSkip([{ + type: 'suite', + attributes: { + suite: `${featuresPath}not-existing.feature` + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = payload.events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) + const testModule = events.find(event => event.type === 'test_module_end').content assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) + assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: false - }) - - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }]) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + }) + if (!isAgentless) { + context('if the agent is not event platform proxy compatible', () => { + it('does not do any intelligent test runner request', (done) => { + receiver.setInfoResponse({ endpoints: [] }) + + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request search_commits') + done(error) + }, ({ url }) => url === '/api/v2/git/repository/search_commits') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') + receiver.assertPayloadReceived(() => { + const error = new Error('should not request setting') + done(error) + }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') + + receiver.assertPayloadReceived(({ payload }) => { + const testSpans = payload.flatMap(trace => trace) + const resourceNames = testSpans.map(span => span.resource) + + assert.includeMembers(resourceNames, + [ + `${featuresPath}farewell.feature.Say farewell`, + `${featuresPath}greetings.feature.Say greetings`, + `${featuresPath}greetings.feature.Say yeah`, + `${featuresPath}greetings.feature.Say yo`, + `${featuresPath}greetings.feature.Say skip` + ] + ) + }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - receiver.assertPayloadReceived(({ payload }) => { - const eventTypes = payload.events.map(event => event.type) - // because they are not skipped - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - const numSuites = eventTypes.reduce( - (acc, type) => type === 'test_suite_end' ? acc + 1 : acc, 0 + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: getCiVisEvpProxyConfig(receiver.port), + stdio: 'inherit' + } ) - assert.equal(numSuites, 2) - }, ({ url }) => url.endsWith('/api/v2/citestcycle')).then(() => done()).catch(done) + }) + }) + } + it('reports itr_correlation_id in test suites', (done) => { + const itrCorrelationId = '4321' + receiver.setItrCorrelationId(itrCorrelationId) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + testSuites.forEach(testSuite => { + assert.equal(testSuite.itr_correlation_id, itrCorrelationId) + }) + }, 25000) + + childProcess = exec( + runTestsWithCoverageCommand, + { + cwd, + env: envVars, + stdio: 'inherit' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisAgentlessConfig(receiver.port), - stdio: 'inherit' + context('early flake detection', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) + } }) - it('does not skip suites if suite is marked as unskippable', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) - - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } - }, - { - type: 'suite', - attributes: { - suite: `${featuresPath}greetings.feature` - } + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_FORCED_RUN, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_FORCED_RUN, 'true') - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ).content - const forcedToRunSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ).content - - assert.propertyVal(skippedSuite.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.meta, TEST_ITR_FORCED_RUN) - - assert.propertyVal(forcedToRunSuite.meta, TEST_STATUS, 'fail') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.propertyVal(forcedToRunSuite.meta, TEST_ITR_FORCED_RUN, 'true') - }, 25000) + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + // Test name does not change + newTests.forEach(test => { + assert.equal(test.meta[TEST_NAME], 'Say whatever') + }) }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('only sets forced to run if suite was going to be skipped by ITR', (done) => { - receiver.setSettings({ - itr_enabled: true, - code_coverage: true, - tests_skipping: true - }) + }) - receiver.setSuitesToSkip([ - { - type: 'suite', - attributes: { - suite: `${featuresPath}farewell.feature` - } + it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ]) - - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const suites = events.filter(event => event.type === 'test_suite_end') - - assert.equal(suites.length, 2) - - const testSession = events.find(event => event.type === 'test_session_end').content - const testModule = events.find(event => event.type === 'test_session_end').content - - assert.propertyVal(testSession.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testSession.meta, TEST_ITR_FORCED_RUN) - assert.propertyVal(testModule.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(testModule.meta, TEST_ITR_FORCED_RUN) - - const skippedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/farewell.feature' - ) - const failedSuite = suites.find( - event => event.content.resource === 'test_suite.ci-visibility/features/greetings.feature' - ) - - assert.propertyVal(skippedSuite.content.meta, TEST_STATUS, 'skip') - assert.notProperty(skippedSuite.content.meta, TEST_ITR_UNSKIPPABLE) - assert.notProperty(skippedSuite.content.meta, TEST_ITR_FORCED_RUN) + } + }) - assert.propertyVal(failedSuite.content.meta, TEST_STATUS, 'fail') - assert.propertyVal(failedSuite.content.meta, TEST_ITR_UNSKIPPABLE, 'true') - assert.notProperty(failedSuite.content.meta, TEST_ITR_FORCED_RUN) - }, 25000) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + // new tests are not detected + assert.equal(newTests.length, 0) }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } }) - it('sets _dd.ci.itr.tests_skipped to false if the received suite is not skipped', (done) => { - receiver.setSuitesToSkip([{ - type: 'suite', - attributes: { - suite: `${featuresPath}not-existing.feature` - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testSession.metrics, TEST_ITR_SKIPPING_COUNT, 0) - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'false') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - assert.propertyVal(testModule.metrics, TEST_ITR_SKIPPING_COUNT, 0) - }, 25000) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: { ...envVars, DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - if (!isAgentless) { - context('if the agent is not event platform proxy compatible', () => { - it('does not do any intelligent test runner request', (done) => { - receiver.setInfoResponse({ endpoints: [] }) - - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request search_commits') - done(error) - }, ({ url }) => url === '/api/v2/git/repository/search_commits') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/api/v2/libraries/tests/services/setting') - receiver.assertPayloadReceived(() => { - const error = new Error('should not request setting') - done(error) - }, ({ url }) => url === '/evp_proxy/v2/api/v2/libraries/tests/services/setting') - - receiver.assertPayloadReceived(({ payload }) => { - const testSpans = payload.flatMap(trace => trace) - const resourceNames = testSpans.map(span => span.resource) - - assert.includeMembers(resourceNames, - [ - `${featuresPath}farewell.feature.Say farewell`, - `${featuresPath}greetings.feature.Say greetings`, - `${featuresPath}greetings.feature.Say yeah`, - `${featuresPath}greetings.feature.Say yo`, - `${featuresPath}greetings.feature.Say skip` - ] - ) - }, ({ url }) => url === '/v0.4/traces').then(() => done()).catch(done) - - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: getCiVisEvpProxyConfig(receiver.port), - stdio: 'inherit' - } - ) - }) - }) - } - it('reports itr_correlation_id in test suites', (done) => { - const itrCorrelationId = '4321' - receiver.setItrCorrelationId(itrCorrelationId) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - testSuites.forEach(testSuite => { - assert.equal(testSuite.itr_correlation_id, itrCorrelationId) - }) - }, 25000) + }) - childProcess = exec( - runTestsWithCoverageCommand, - { - cwd, - env: envVars, - stdio: 'inherit' + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + } }) - }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) - context('early flake detection', () => { - it('retries new tests', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - // "cucumber.ci-visibility/features/farewell.feature.Say" whatever will be considered new - receiver.setKnownTests( - { - cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] - } - } - ) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) - const newTests = tests.filter(test => - test.resource === 'ci-visibility/features/farewell.feature.Say whatever' - ) - newTests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) - const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - // all but one has been retried - assert.equal( - newTests.length - 1, - retriedTests.length - ) - assert.equal(retriedTests.length, NUM_RETRIES_EFD) - // Test name does not change - newTests.forEach(test => { - assert.equal(test.meta[TEST_NAME], 'Say whatever') - }) + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + }) + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) - }) - it('is disabled if DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED is false', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - const tests = events.filter(event => event.type === 'test').map(event => event.content) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - // new tests are not detected - assert.equal(newTests.length, 0) - }) - // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new - receiver.setKnownTests({ - cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] - } + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) }) - childProcess = exec( - runTestsCommand, - { - cwd, - env: { ...envVars, DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED: 'false' }, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) - }) + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) + eventsPromise.then(() => { + done() + }).catch(done) }) - it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } - }) - // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new - receiver.setKnownTests({}) + }) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + } + }) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - tests.forEach(test => { - assert.propertyVal(test.meta, TEST_IS_NEW, 'true') - }) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) + }) - // (1 original run + 3 retries) / 2 - assert.equal(failedAttempts.length, 2) - assert.equal(passedAttempts.length, 2) - }) + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) - childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature', - { - cwd, - env: envVars, - stdio: 'pipe' + it('does not run EFD if the known tests request fails', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD } - ) - childProcess.on('exit', (exitCode) => { - assert.equal(exitCode, 0) - eventsPromise.then(() => { - done() - }).catch(done) - }) + } }) - it('does not retry tests that are skipped', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD - } - } + receiver.setKnownTestsResponseCode(500) + receiver.setKnownTests({}) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 6) + const newTests = tests.filter(test => + test.meta[TEST_IS_NEW] === 'true' + ) + assert.equal(newTests.length, 0) }) - // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new - // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new - receiver.setKnownTests({ + + childProcess = exec( + runTestsCommand, + { cwd, env: envVars, stdio: 'pipe' } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 + } + }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { cucumber: { - 'ci-visibility/features/farewell.feature': ['Say farewell'], - 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] } - }) + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) - const skippedNewTest = tests.filter(test => - test.resource === 'ci-visibility/features/greetings.feature.Say skip' - ) - // not retried - assert.equal(skippedNewTest.length, 1) - }) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) - childProcess = exec( - runTestsCommand, - { - cwd, - env: envVars, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) }) + + childProcess = exec( + runTestsCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) }) - it('does not run EFD if the known tests request fails', (done) => { - const NUM_RETRIES_EFD = 3 - receiver.setSettings({ - itr_enabled: false, - code_coverage: false, - tests_skipping: false, - early_flake_detection: { - enabled: true, - slow_test_retries: { - '5s': NUM_RETRIES_EFD + }) + + if (version !== '7.0.0') { // EFD in parallel mode only supported from cucumber>=11 + context('parallel mode', () => { + it('retries new tests', (done) => { + const NUM_RETRIES_EFD = 3 + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } - } - }) - receiver.setKnownTestsResponseCode(500) - receiver.setKnownTests({}) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { - const events = payloads.flatMap(({ payload }) => payload.events) + }) + // cucumber.ci-visibility/features/farewell.feature.Say whatever will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) - const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') - assert.equal(tests.length, 6) - const newTests = tests.filter(test => - test.meta[TEST_IS_NEW] === 'true' - ) - assert.equal(newTests.length, 0) - }) + const tests = events.filter(event => event.type === 'test').map(event => event.content) - childProcess = exec( - runTestsCommand, - { cwd, env: envVars, stdio: 'pipe' } - ) + const newTests = tests.filter(test => + test.resource === 'ci-visibility/features/farewell.feature.Say whatever' + ) + newTests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + // Test name does not change + assert.propertyVal(test.meta, TEST_NAME, 'Say whatever') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + // all but one has been retried + assert.equal( + newTests.length - 1, + retriedTests.length + ) + assert.equal(retriedTests.length, NUM_RETRIES_EFD) + }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + childProcess = exec( + parallelModeCommand, + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) }) - }) - }) - if (version === 'latest') { // flaky test retries only supported from >=8.0.0 - context('flaky test retries', () => { - it('can retry failed tests', (done) => { + it('retries flaky tests and sets exit code to 0 as long as one attempt passes', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } } }) + // Tests in "cucumber.ci-visibility/features-flaky/flaky.feature" will be considered new + receiver.setKnownTests({}) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') const tests = events.filter(event => event.type === 'test').map(event => event.content) + const testSuites = events + .filter(event => event.type === 'test_suite_end').map(event => event.content) - // 2 failures and 1 passed attempt - assert.equal(tests.length, 3) + tests.forEach(test => { + assert.propertyVal(test.meta, TEST_IS_NEW, 'true') + assert.propertyVal(test.meta, CUCUMBER_IS_PARALLEL, 'true') + }) - const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(failedTests.length, 2) - const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - assert.equal(passedTests.length, 1) + // All test suites pass, even though there are failed tests + testSuites.forEach(testSuite => { + assert.propertyVal(testSuite.meta, TEST_STATUS, 'pass') + }) - // All but the first one are retries - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 2) + const failedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + const passedAttempts = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + + // (1 original run + 3 retries) / 2 + assert.equal(failedAttempts.length, 2) + assert.equal(passedAttempts.length, 2) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + './node_modules/.bin/cucumber-js ci-visibility/features-flaky/*.feature --parallel 2', { cwd, env: envVars, @@ -1097,44 +1227,60 @@ versions.forEach(version => { } ) - childProcess.on('exit', () => { + childProcess.on('exit', (exitCode) => { + assert.equal(exitCode, 0) eventsPromise.then(() => { done() }).catch(done) }) }) - it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + it('bails out of EFD if the percentage of new tests is too high', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + }, + faulty_session_threshold: 0 } }) + // tests in cucumber.ci-visibility/features/farewell.feature will be considered new + receiver.setKnownTests( + { + cucumber: { + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo', 'Say skip'] + } + } + ) const eventsPromise = receiver .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.notProperty(testSession.meta, TEST_EARLY_FLAKE_ENABLED) + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') + const tests = events.filter(event => event.type === 'test').map(event => event.content) - assert.equal(tests.length, 1) + const newTests = tests.filter(test => test.meta[TEST_IS_NEW] === 'true') + assert.equal(newTests.length, 0) - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + const retriedTests = newTests.filter(test => test.meta[TEST_IS_RETRY] === 'true') assert.equal(retriedTests.length, 0) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + parallelModeCommand, { cwd, - env: { - ...envVars, - DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' - }, + env: envVars, stdio: 'pipe' } ) @@ -1146,14 +1292,25 @@ versions.forEach(version => { }) }) - it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + it('does not retry tests that are skipped', (done) => { + const NUM_RETRIES_EFD = 3 receiver.setSettings({ itr_enabled: false, code_coverage: false, tests_skipping: false, - flaky_test_retries_enabled: true, early_flake_detection: { - enabled: false + enabled: true, + slow_test_retries: { + '5s': NUM_RETRIES_EFD + } + } + }) + // "cucumber.ci-visibility/features/farewell.feature.Say whatever" will be considered new + // "cucumber.ci-visibility/features/greetings.feature.Say skip" will be considered new + receiver.setKnownTests({ + cucumber: { + 'ci-visibility/features/farewell.feature': ['Say farewell'], + 'ci-visibility/features/greetings.feature': ['Say greetings', 'Say yeah', 'Say yo'] } }) @@ -1161,33 +1318,26 @@ versions.forEach(version => { .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_EARLY_FLAKE_ENABLED, 'true') + assert.propertyVal(testSession.meta, CUCUMBER_IS_PARALLEL, 'true') const tests = events.filter(event => event.type === 'test').map(event => event.content) - // 2 failures - assert.equal(tests.length, 2) - - const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') - assert.equal(failedTests.length, 2) - const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') - assert.equal(passedTests.length, 0) - - // All but the first one are retries - const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') - assert.equal(retriedTests.length, 1) + const skippedNewTest = tests.filter(test => + test.resource === 'ci-visibility/features/greetings.feature.Say skip' + ) + // not retried + assert.equal(skippedNewTest.length, 1) }) childProcess = exec( - './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + parallelModeCommand, { cwd, - env: { - ...envVars, - DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 - }, + env: envVars, stdio: 'pipe' } ) - childProcess.on('exit', () => { eventsPromise.then(() => { done() @@ -1197,55 +1347,235 @@ versions.forEach(version => { }) } }) - }) - it('correctly calculates test code owners when working directory is not repository root', (done) => { - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) + if (version === 'latest') { // flaky test retries only supported from >=8.0.0 + context('flaky test retries', () => { + it('can retry failed tests', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures and 1 passed attempt + assert.equal(tests.length, 3) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 1) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 2) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: envVars, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('is disabled if DD_CIVISIBILITY_FLAKY_RETRY_ENABLED is false', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.equal(tests.length, 1) + + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 0) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_ENABLED: 'false' + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('retries DD_CIVISIBILITY_FLAKY_RETRY_COUNT times', (done) => { + receiver.setSettings({ + itr_enabled: false, + code_coverage: false, + tests_skipping: false, + flaky_test_retries_enabled: true, + early_flake_detection: { + enabled: false + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + // 2 failures + assert.equal(tests.length, 2) + + const failedTests = tests.filter(test => test.meta[TEST_STATUS] === 'fail') + assert.equal(failedTests.length, 2) + const passedTests = tests.filter(test => test.meta[TEST_STATUS] === 'pass') + assert.equal(passedTests.length, 0) + + // All but the first one are retries + const retriedTests = tests.filter(test => test.meta[TEST_IS_RETRY] === 'true') + assert.equal(retriedTests.length, 1) + }) + + childProcess = exec( + './node_modules/.bin/cucumber-js ci-visibility/features-retry/*.feature', + { + cwd, + env: { + ...envVars, + DD_CIVISIBILITY_FLAKY_RETRY_COUNT: 1 + }, + stdio: 'pipe' + } + ) - const test = events.find(event => event.type === 'test').content - const testSuite = events.find(event => event.type === 'test_suite_end').content - // The test is in a subproject - assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) - assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) - assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) }) + } + }) + }) - childProcess = exec( - 'node ../../node_modules/.bin/cucumber-js features/*.feature', - { - cwd: `${cwd}/ci-visibility/subproject`, - env: { - ...getCiVisAgentlessConfig(receiver.port) - }, - stdio: 'inherit' - } - ) + it('correctly calculates test code owners when working directory is not repository root', (done) => { + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + + const test = events.find(event => event.type === 'test').content + const testSuite = events.find(event => event.type === 'test_suite_end').content + // The test is in a subproject + assert.notEqual(test.meta[TEST_SOURCE_FILE], test.meta[TEST_SUITE]) + assert.equal(test.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + assert.equal(testSuite.meta[TEST_CODE_OWNERS], JSON.stringify(['@datadog-dd-trace-js'])) + }) - childProcess.on('exit', () => { - eventsPromise.then(() => { - done() - }).catch(done) + childProcess = exec( + 'node ../../node_modules/.bin/cucumber-js features/*.feature', + { + cwd: `${cwd}/ci-visibility/subproject`, + env: { + ...getCiVisAgentlessConfig(receiver.port) + }, + stdio: 'inherit' + } + ) + + childProcess.on('exit', () => { + eventsPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('takes into account untested files if "all" is passed to nyc', (done) => { + const linesPctMatchRegex = /Lines\s*:\s*([\d.]+)%/ + let linesPctMatch + let linesPctFromNyc = 0 + let codeCoverageWithUntestedFiles = 0 + let codeCoverageWithoutUntestedFiles = 0 + + let eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] }) + + childProcess = exec( + './node_modules/nyc/bin/nyc.js --all -r=text-summary --nycrc-path ./my-nyc.config.js ' + + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NYC_INCLUDE: JSON.stringify( + [ + 'ci-visibility/features/**', + 'ci-visibility/features-esm/**' + ] + ) + }, + stdio: 'inherit' + } + ) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() }) - it('takes into account untested files if "all" is passed to nyc', (done) => { - const linesPctMatchRegex = /Lines\s*:\s*([\d.]+)%/ - let linesPctMatch - let linesPctFromNyc = 0 - let codeCoverageWithUntestedFiles = 0 - let codeCoverageWithoutUntestedFiles = 0 + childProcess.on('exit', () => { + linesPctMatch = testOutput.match(linesPctMatchRegex) + linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null - let eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - codeCoverageWithUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] - }) + assert.equal( + linesPctFromNyc, + codeCoverageWithUntestedFiles, + 'nyc --all output does not match the reported coverage' + ) + // reset test output for next test session + testOutput = '' + // we run the same tests without the all flag childProcess = exec( - './node_modules/nyc/bin/nyc.js --all -r=text-summary --nycrc-path ./my-nyc.config.js ' + + './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', { cwd, @@ -1262,6 +1592,13 @@ versions.forEach(version => { } ) + eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testSession = events.find(event => event.type === 'test_session_end').content + codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] + }) + childProcess.stdout.on('data', (chunk) => { testOutput += chunk.toString() }) @@ -1275,60 +1612,14 @@ versions.forEach(version => { assert.equal( linesPctFromNyc, - codeCoverageWithUntestedFiles, - 'nyc --all output does not match the reported coverage' - ) - - // reset test output for next test session - testOutput = '' - // we run the same tests without the all flag - childProcess = exec( - './node_modules/nyc/bin/nyc.js -r=text-summary --nycrc-path ./my-nyc.config.js ' + - 'node ./node_modules/.bin/cucumber-js ci-visibility/features/*.feature', - { - cwd, - env: { - ...getCiVisAgentlessConfig(receiver.port), - NYC_INCLUDE: JSON.stringify( - [ - 'ci-visibility/features/**', - 'ci-visibility/features-esm/**' - ] - ) - }, - stdio: 'inherit' - } + codeCoverageWithoutUntestedFiles, + 'nyc output does not match the reported coverage (no --all flag)' ) - eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const testSession = events.find(event => event.type === 'test_session_end').content - codeCoverageWithoutUntestedFiles = testSession.metrics[TEST_CODE_COVERAGE_LINES_PCT] - }) - - childProcess.stdout.on('data', (chunk) => { - testOutput += chunk.toString() - }) - childProcess.stderr.on('data', (chunk) => { - testOutput += chunk.toString() - }) - - childProcess.on('exit', () => { - linesPctMatch = testOutput.match(linesPctMatchRegex) - linesPctFromNyc = linesPctMatch ? Number(linesPctMatch[1]) : null - - assert.equal( - linesPctFromNyc, - codeCoverageWithoutUntestedFiles, - 'nyc output does not match the reported coverage (no --all flag)' - ) - - eventsPromise.then(() => { - assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) - done() - }).catch(done) - }) + eventsPromise.then(() => { + assert.isAbove(codeCoverageWithoutUntestedFiles, codeCoverageWithUntestedFiles) + done() + }).catch(done) }) }) }) diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index abe46f21aaf..0f84d717381 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -35,7 +35,8 @@ const { mergeCoverage, fromCoverageMapToCoverage, getTestSuitePath, - CUCUMBER_WORKER_TRACE_PAYLOAD_CODE + CUCUMBER_WORKER_TRACE_PAYLOAD_CODE, + getIsFaultyEarlyFlakeDetection } = require('../../dd-trace/src/plugins/util/test') const isMarkedAsUnskippable = (pickle) => { @@ -51,6 +52,7 @@ const patched = new WeakSet() const lastStatusByPickleId = new Map() const numRetriesByPickleId = new Map() const numAttemptToAsyncResource = new Map() +const newTestsByTestFullname = new Map() let eventDataCollector = null let pickleByFile = {} @@ -65,6 +67,8 @@ let isUnskippable = false let isSuitesSkippingEnabled = false let isEarlyFlakeDetectionEnabled = false let earlyFlakeDetectionNumRetries = 0 +let earlyFlakeDetectionFaultyThreshold = 0 +let isEarlyFlakeDetectionFaulty = false let isFlakyTestRetriesEnabled = false let numTestRetries = 0 let knownTests = [] @@ -351,6 +355,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin isEarlyFlakeDetectionEnabled = configurationResponse.libraryConfig?.isEarlyFlakeDetectionEnabled earlyFlakeDetectionNumRetries = configurationResponse.libraryConfig?.earlyFlakeDetectionNumRetries + earlyFlakeDetectionFaultyThreshold = configurationResponse.libraryConfig?.earlyFlakeDetectionFaultyThreshold isSuitesSkippingEnabled = configurationResponse.libraryConfig?.isSuitesSkippingEnabled isFlakyTestRetriesEnabled = configurationResponse.libraryConfig?.isFlakyTestRetriesEnabled numTestRetries = configurationResponse.libraryConfig?.flakyTestRetriesCount @@ -397,6 +402,18 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin pickleByFile = isCoordinator ? getPickleByFileNew(this) : getPickleByFile(this) + if (isEarlyFlakeDetectionEnabled) { + const isFaulty = getIsFaultyEarlyFlakeDetection( + Object.keys(pickleByFile), + knownTests.cucumber || {}, + earlyFlakeDetectionFaultyThreshold + ) + if (isFaulty) { + isEarlyFlakeDetectionEnabled = false + isEarlyFlakeDetectionFaulty = true + } + } + const processArgv = process.argv.slice(2).join(' ') const command = process.env.npm_lifecycle_script || `cucumber-js ${processArgv}` @@ -443,6 +460,7 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin hasUnskippableSuites: isUnskippable, hasForcedToRunSuites: isForcedToRun, isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, isParallel }) }) @@ -451,7 +469,9 @@ function getWrappedStart (start, frameworkVersion, isParallel = false, isCoordin } } -function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { +// Generates suite start and finish events in the main process. +// Handles EFD in both the main process and the worker process. +function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion = false, isWorker = false) { return async function () { let pickle if (isNewerCucumberVersion) { @@ -463,7 +483,8 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { const testFileAbsolutePath = pickle.uri const testSuitePath = getTestSuitePath(testFileAbsolutePath, process.cwd()) - if (!pickleResultByFile[testFileAbsolutePath]) { // first test in suite + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && !pickleResultByFile[testFileAbsolutePath]) { // first test in suite isUnskippable = isMarkedAsUnskippable(pickle) isForcedToRun = isUnskippable && skippableSuites.includes(testSuitePath) @@ -519,8 +540,9 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { pickleResultByFile[testFileAbsolutePath].push(testStatus) } - // last test in suite - if (pickleResultByFile[testFileAbsolutePath].length === pickleByFile[testFileAbsolutePath].length) { + // If it's a worker, suite events are handled in `getWrappedParseWorkerMessage` + if (!isWorker && pickleResultByFile[testFileAbsolutePath].length === pickleByFile[testFileAbsolutePath].length) { + // last test in suite const testSuiteStatus = getSuiteStatusFromTestStatuses(pickleResultByFile[testFileAbsolutePath]) if (global.__coverage__) { const coverageFiles = getCoveredFilenamesFromCoverage(global.__coverage__) @@ -539,7 +561,7 @@ function getWrappedRunTestCase (runTestCaseFunction, isNewerCucumberVersion) { testSuiteFinishCh.publish({ status: testSuiteStatus, testSuitePath }) } - if (isNewerCucumberVersion && isNew && isEarlyFlakeDetectionEnabled) { + if (isNewerCucumberVersion && isEarlyFlakeDetectionEnabled && isNew) { return shouldBePassedByEFD } @@ -618,16 +640,40 @@ function getWrappedParseWorkerMessage (parseWorkerMessageFunction, isNewVersion) pickle = testCase.pickle } - // TODO: can we get error message? const { status } = getStatusFromResultLatest(worstTestStepResult) + let isNew = false + + if (isEarlyFlakeDetectionEnabled) { + isNew = isNewTest(pickle.uri, pickle.name) + } const testFileAbsolutePath = pickle.uri const finished = pickleResultByFile[testFileAbsolutePath] - finished.push(status) + + if (isNew) { + const testFullname = `${pickle.uri}:${pickle.name}` + let testStatuses = newTestsByTestFullname.get(testFullname) + if (!testStatuses) { + testStatuses = [status] + newTestsByTestFullname.set(testFullname, testStatuses) + } else { + testStatuses.push(status) + } + // We have finished all retries + if (testStatuses.length === earlyFlakeDetectionNumRetries + 1) { + const newTestFinalStatus = getTestStatusFromRetries(testStatuses) + // we only push to `finished` if the retries have finished + finished.push(newTestFinalStatus) + } + } else { + // TODO: can we get error message? + const finished = pickleResultByFile[testFileAbsolutePath] + finished.push(status) + } if (finished.length === pickleByFile[testFileAbsolutePath].length) { testSuiteFinishCh.publish({ - status: getSuiteStatusFromTestStatuses(finished), // maybe tests themselves can add to this list + status: getSuiteStatusFromTestStatuses(finished), testSuitePath: getTestSuitePath(testFileAbsolutePath, process.cwd()) }) } @@ -645,7 +691,6 @@ addHook({ }, pickleHook) // Test start / finish for newer versions. The only hook executed in workers when in parallel mode - addHook({ name: '@cucumber/cucumber', versions: ['>=7.3.0'], @@ -701,15 +746,19 @@ addHook({ }) // >=11.0.0 hooks -// `getWrappedRunTestCase` generates suite start and finish events and handles EFD. +// `getWrappedRunTestCase` does two things: +// - generates suite start and finish events in the main process, +// - handles EFD in both the main process and the worker process. addHook({ name: '@cucumber/cucumber', versions: ['>=11.0.0'], file: 'lib/runtime/worker.js' }, (workerPackage) => { - if (!process.env.CUCUMBER_WORKER_ID) { - shimmer.wrap(workerPackage.Worker.prototype, 'runTestCase', runTestCase => getWrappedRunTestCase(runTestCase, true)) - } + shimmer.wrap( + workerPackage.Worker.prototype, + 'runTestCase', + runTestCase => getWrappedRunTestCase(runTestCase, true, !!process.env.CUCUMBER_WORKER_ID) + ) return workerPackage }) @@ -740,8 +789,9 @@ addHook({ return eventDataCollectorPackage }) -// Only executed in parallel mode for >=11. +// Only executed in parallel mode for >=11, in the main process. // `getWrappedParseWorkerMessage` generates suite start and finish events +// In `startWorker` we pass early flake detection info to the worker. addHook({ name: '@cucumber/cucumber', versions: ['>=11.0.0'], @@ -752,5 +802,37 @@ addHook({ 'parseWorkerMessage', parseWorkerMessage => getWrappedParseWorkerMessage(parseWorkerMessage, true) ) + // EFD in parallel mode only supported in >=11.0.0 + shimmer.wrap(adapterPackage.ChildProcessAdapter.prototype, 'startWorker', startWorker => function () { + if (isEarlyFlakeDetectionEnabled) { + this.options.worldParameters._ddKnownTests = knownTests + this.options.worldParameters._ddEarlyFlakeDetectionNumRetries = earlyFlakeDetectionNumRetries + } + + return startWorker.apply(this, arguments) + }) return adapterPackage }) + +// Hook executed in the worker process when in parallel mode. +// In this hook we read the information passed in `worldParameters` and make it available for +// `getWrappedRunTestCase`. +addHook({ + name: '@cucumber/cucumber', + versions: ['>=11.0.0'], + file: 'lib/runtime/parallel/worker.js' +}, (workerPackage) => { + shimmer.wrap( + workerPackage.ChildProcessWorker.prototype, + 'initialize', + initialize => async function () { + await initialize.apply(this, arguments) + isEarlyFlakeDetectionEnabled = !!this.options.worldParameters._ddKnownTests + if (isEarlyFlakeDetectionEnabled) { + knownTests = this.options.worldParameters._ddKnownTests + earlyFlakeDetectionNumRetries = this.options.worldParameters._ddEarlyFlakeDetectionNumRetries + } + } + ) + return workerPackage +}) diff --git a/packages/datadog-plugin-cucumber/src/index.js b/packages/datadog-plugin-cucumber/src/index.js index 7aaf264d763..d24f97c33e6 100644 --- a/packages/datadog-plugin-cucumber/src/index.js +++ b/packages/datadog-plugin-cucumber/src/index.js @@ -17,6 +17,7 @@ const { ITR_CORRELATION_ID, TEST_SOURCE_FILE, TEST_EARLY_FLAKE_ENABLED, + TEST_EARLY_FLAKE_ABORT_REASON, TEST_IS_NEW, TEST_IS_RETRY, TEST_SUITE_ID, @@ -79,6 +80,7 @@ class CucumberPlugin extends CiPlugin { hasUnskippableSuites, hasForcedToRunSuites, isEarlyFlakeDetectionEnabled, + isEarlyFlakeDetectionFaulty, isParallel }) => { const { isSuitesSkippingEnabled, isCodeCoverageEnabled } = this.libraryConfig || {} @@ -99,6 +101,9 @@ class CucumberPlugin extends CiPlugin { if (isEarlyFlakeDetectionEnabled) { this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ENABLED, 'true') } + if (isEarlyFlakeDetectionFaulty) { + this.testSessionSpan.setTag(TEST_EARLY_FLAKE_ABORT_REASON, 'faulty') + } if (isParallel) { this.testSessionSpan.setTag(CUCUMBER_IS_PARALLEL, 'true') } From 3c2fedc802f604889ec0bd760bf3841a8ab88a3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 30 Sep 2024 14:35:22 +0200 Subject: [PATCH 02/27] Do not run fastify >=5 plugin tests in node <=18 (#4737) Co-authored-by: Ugaitz Urien --- packages/datadog-plugin-cucumber/test/index.spec.js | 2 +- packages/datadog-plugin-fastify/test/index.spec.js | 5 ++++- .../test/integration-test/client.spec.js | 7 +++++-- packages/datadog-plugin-undici/test/index.spec.js | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/datadog-plugin-cucumber/test/index.spec.js b/packages/datadog-plugin-cucumber/test/index.spec.js index e9ef915dab8..a43a2a53509 100644 --- a/packages/datadog-plugin-cucumber/test/index.spec.js +++ b/packages/datadog-plugin-cucumber/test/index.spec.js @@ -56,7 +56,7 @@ describe('Plugin', function () { let Cucumber this.timeout(10000) withVersions('cucumber', '@cucumber/cucumber', (version, _, specificVersion) => { - if ((NODE_MAJOR <= 16) && semver.satisfies(specificVersion, '>=10')) return + if (NODE_MAJOR <= 16 && semver.satisfies(specificVersion, '>=10')) return afterEach(() => { // > If you want to run tests multiple times, you may need to clear Node's require cache diff --git a/packages/datadog-plugin-fastify/test/index.spec.js b/packages/datadog-plugin-fastify/test/index.spec.js index 33b1430f98c..6b20e58a728 100644 --- a/packages/datadog-plugin-fastify/test/index.spec.js +++ b/packages/datadog-plugin-fastify/test/index.spec.js @@ -5,6 +5,7 @@ const axios = require('axios') const semver = require('semver') const { ERROR_MESSAGE, ERROR_STACK, ERROR_TYPE } = require('../../dd-trace/src/constants') const agent = require('../../dd-trace/test/plugins/agent') +const { NODE_MAJOR } = require('../../../version') const host = 'localhost' @@ -14,7 +15,7 @@ describe('Plugin', () => { let app describe('fastify', () => { - withVersions('fastify', 'fastify', version => { + withVersions('fastify', 'fastify', (version, _, specificVersion) => { beforeEach(() => { tracer = require('../../dd-trace') }) @@ -25,6 +26,8 @@ describe('Plugin', () => { withExports('fastify', version, ['default', 'fastify'], '>=3', getExport => { describe('without configuration', () => { + if (NODE_MAJOR <= 18 && semver.satisfies(specificVersion, '>=5')) return + before(() => { return agent.load(['fastify', 'find-my-way', 'http'], [{}, {}, { client: false }]) }) diff --git a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js index 1ccb9791dc6..6a04cf6912b 100644 --- a/packages/datadog-plugin-fastify/test/integration-test/client.spec.js +++ b/packages/datadog-plugin-fastify/test/integration-test/client.spec.js @@ -1,5 +1,5 @@ 'use strict' - +const semver = require('semver') const { FakeAgent, createSandbox, @@ -8,6 +8,7 @@ const { spawnPluginIntegrationTestProc } = require('../../../../integration-tests/helpers') const { assert } = require('chai') +const { NODE_MAJOR } = require('../../../../version') describe('esm', () => { let agent @@ -15,7 +16,9 @@ describe('esm', () => { let sandbox // skip older versions of fastify due to syntax differences - withVersions('fastify', 'fastify', '>=3', version => { + withVersions('fastify', 'fastify', '>=3', (version, _, specificVersion) => { + if (NODE_MAJOR <= 18 && semver.satisfies(specificVersion, '>=5')) return + before(async function () { this.timeout(20000) sandbox = await createSandbox([`'fastify@${version}'`], false, diff --git a/packages/datadog-plugin-undici/test/index.spec.js b/packages/datadog-plugin-undici/test/index.spec.js index 1224ead7f7f..03541e0eb7c 100644 --- a/packages/datadog-plugin-undici/test/index.spec.js +++ b/packages/datadog-plugin-undici/test/index.spec.js @@ -23,7 +23,7 @@ describe('Plugin', () => { describe('undici-fetch', () => { withVersions('undici', 'undici', version => { const specificVersion = require(`../../../versions/undici@${version}`).version() - if ((NODE_MAJOR <= 16) && semver.satisfies(specificVersion, '>=6')) return + if (NODE_MAJOR <= 16 && semver.satisfies(specificVersion, '>=6')) return function server (app, listener) { const server = require('http').createServer(app) From a6ab981c60e57ae4026bef00aed6c08465b46807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 30 Sep 2024 16:57:03 +0200 Subject: [PATCH 03/27] Bump import-in-the-middle to 1.11.2 (#4740) --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 49958f4861c..75f97975880 100644 --- a/package.json +++ b/package.json @@ -87,7 +87,7 @@ "crypto-randomuuid": "^1.0.0", "dc-polyfill": "^0.1.4", "ignore": "^5.2.4", - "import-in-the-middle": "1.11.0", + "import-in-the-middle": "1.11.2", "int64-buffer": "^0.1.9", "istanbul-lib-coverage": "3.2.0", "jest-docblock": "^29.7.0", diff --git a/yarn.lock b/yarn.lock index ca8b325ec9f..cf7cba3f3f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2555,10 +2555,10 @@ import-fresh@^3.2.1: parent-module "^1.0.0" resolve-from "^4.0.0" -import-in-the-middle@1.11.0: - version "1.11.0" - resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.0.tgz#a94c4925b8da18256cde3b3b7b38253e6ca5e708" - integrity sha512-5DimNQGoe0pLUHbR9qK84iWaWjjbsxiqXnw6Qz64+azRgleqv9k2kTt5fw7QsOpmaGYtuxxursnPPsnTKEx10Q== +import-in-the-middle@1.11.2: + version "1.11.2" + resolved "https://registry.yarnpkg.com/import-in-the-middle/-/import-in-the-middle-1.11.2.tgz#dd848e72b63ca6cd7c34df8b8d97fc9baee6174f" + integrity sha512-gK6Rr6EykBcc6cVWRSBR5TWf8nn6hZMYSRYqCcHa0l0d1fPK7JSYo6+Mlmck76jIX9aL/IZ71c06U2VpFwl1zA== dependencies: acorn "^8.8.2" acorn-import-attributes "^1.9.5" From fcd3ab8244800e64b01bdd746933812dada90ad5 Mon Sep 17 00:00:00 2001 From: William Conti <58711692+wconti27@users.noreply.github.com> Date: Mon, 30 Sep 2024 15:07:19 -0400 Subject: [PATCH 04/27] fix: still set dsm consume checkpoint when no DSM parent context is available (#4741) * fix dsm checkpointing when no parent context is available when consuming --- packages/datadog-plugin-amqplib/src/consumer.js | 4 +--- .../src/services/kinesis.js | 2 +- .../datadog-plugin-aws-sdk/src/services/sqs.js | 2 +- .../src/batch-consumer.js | 3 +-- packages/datadog-plugin-kafkajs/src/consumer.js | 3 +-- packages/datadog-plugin-rhea/src/consumer.js | 4 +--- packages/dd-trace/src/data_streams_context.js | 3 +++ packages/dd-trace/src/datastreams/pathway.js | 17 ++++++++++++----- packages/dd-trace/src/datastreams/processor.js | 6 ++++++ 9 files changed, 27 insertions(+), 17 deletions(-) diff --git a/packages/datadog-plugin-amqplib/src/consumer.js b/packages/datadog-plugin-amqplib/src/consumer.js index da4efb33fd0..92684e3f9dc 100644 --- a/packages/datadog-plugin-amqplib/src/consumer.js +++ b/packages/datadog-plugin-amqplib/src/consumer.js @@ -3,7 +3,6 @@ const { TEXT_MAP } = require('../../../ext/formats') const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') -const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') const { getResourceName } = require('./util') class AmqplibConsumerPlugin extends ConsumerPlugin { @@ -30,8 +29,7 @@ class AmqplibConsumerPlugin extends ConsumerPlugin { }) if ( - this.config.dsmEnabled && message?.properties?.headers && - DsmPathwayCodec.contextExists(message.properties.headers) + this.config.dsmEnabled && message?.properties?.headers ) { const payloadSize = getAmqpMessageSize({ headers: message.properties.headers, content: message.content }) const queue = fields.queue ? fields.queue : fields.routingKey diff --git a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js index 98547c564f8..60802bfc448 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/kinesis.js +++ b/packages/datadog-plugin-aws-sdk/src/services/kinesis.js @@ -113,7 +113,7 @@ class Kinesis extends BaseAwsSdkPlugin { const parsedAttributes = JSON.parse(Buffer.from(record.Data).toString()) if ( - parsedAttributes?._datadog && streamName && DsmPathwayCodec.contextExists(parsedAttributes._datadog) + parsedAttributes?._datadog && streamName ) { const payloadSize = getSizeOrZero(record.Data) this.tracer.decodeDataStreamsContext(parsedAttributes._datadog) diff --git a/packages/datadog-plugin-aws-sdk/src/services/sqs.js b/packages/datadog-plugin-aws-sdk/src/services/sqs.js index 35854ed3c1d..54a3e7e756c 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/sqs.js +++ b/packages/datadog-plugin-aws-sdk/src/services/sqs.js @@ -194,7 +194,7 @@ class Sqs extends BaseAwsSdkPlugin { parsedAttributes = this.parseDatadogAttributes(message.MessageAttributes._datadog) } } - if (parsedAttributes && DsmPathwayCodec.contextExists(parsedAttributes)) { + if (parsedAttributes) { const payloadSize = getHeadersSize({ Body: message.Body, MessageAttributes: message.MessageAttributes diff --git a/packages/datadog-plugin-kafkajs/src/batch-consumer.js b/packages/datadog-plugin-kafkajs/src/batch-consumer.js index 5a531267e9b..8415b037644 100644 --- a/packages/datadog-plugin-kafkajs/src/batch-consumer.js +++ b/packages/datadog-plugin-kafkajs/src/batch-consumer.js @@ -1,6 +1,5 @@ const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') -const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') class KafkajsBatchConsumerPlugin extends ConsumerPlugin { static get id () { return 'kafkajs' } @@ -9,7 +8,7 @@ class KafkajsBatchConsumerPlugin extends ConsumerPlugin { start ({ topic, partition, messages, groupId }) { if (!this.config.dsmEnabled) return for (const message of messages) { - if (!message || !message.headers || !DsmPathwayCodec.contextExists(message.headers)) continue + if (!message || !message.headers) continue const payloadSize = getMessageSize(message) this.tracer.decodeDataStreamsContext(message.headers) this.tracer diff --git a/packages/datadog-plugin-kafkajs/src/consumer.js b/packages/datadog-plugin-kafkajs/src/consumer.js index 420fea10902..84b6a02fdda 100644 --- a/packages/datadog-plugin-kafkajs/src/consumer.js +++ b/packages/datadog-plugin-kafkajs/src/consumer.js @@ -2,7 +2,6 @@ const dc = require('dc-polyfill') const { getMessageSize } = require('../../dd-trace/src/datastreams/processor') -const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') const afterStartCh = dc.channel('dd-trace:kafkajs:consumer:afterStart') @@ -78,7 +77,7 @@ class KafkajsConsumerPlugin extends ConsumerPlugin { 'kafka.partition': partition } }) - if (this.config.dsmEnabled && message?.headers && DsmPathwayCodec.contextExists(message.headers)) { + if (this.config.dsmEnabled && message?.headers) { const payloadSize = getMessageSize(message) this.tracer.decodeDataStreamsContext(message.headers) this.tracer diff --git a/packages/datadog-plugin-rhea/src/consumer.js b/packages/datadog-plugin-rhea/src/consumer.js index 226834885be..56aad8f7b9d 100644 --- a/packages/datadog-plugin-rhea/src/consumer.js +++ b/packages/datadog-plugin-rhea/src/consumer.js @@ -3,7 +3,6 @@ const ConsumerPlugin = require('../../dd-trace/src/plugins/consumer') const { storage } = require('../../datadog-core') const { getAmqpMessageSize } = require('../../dd-trace/src/datastreams/processor') -const { DsmPathwayCodec } = require('../../dd-trace/src/datastreams/pathway') class RheaConsumerPlugin extends ConsumerPlugin { static get id () { return 'rhea' } @@ -34,8 +33,7 @@ class RheaConsumerPlugin extends ConsumerPlugin { if ( this.config.dsmEnabled && - msgObj?.message?.delivery_annotations && - DsmPathwayCodec.contextExists(msgObj.message.delivery_annotations) + msgObj?.message?.delivery_annotations ) { const payloadSize = getAmqpMessageSize( { headers: msgObj.message.delivery_annotations, content: msgObj.message.body } diff --git a/packages/dd-trace/src/data_streams_context.js b/packages/dd-trace/src/data_streams_context.js index 33354920443..e3c62d35e25 100644 --- a/packages/dd-trace/src/data_streams_context.js +++ b/packages/dd-trace/src/data_streams_context.js @@ -1,4 +1,5 @@ const { storage } = require('../../datadog-core') +const log = require('./log') function getDataStreamsContext () { const store = storage.getStore() @@ -6,6 +7,8 @@ function getDataStreamsContext () { } function setDataStreamsContext (dataStreamsContext) { + log.debug(() => `Setting new DSM Context: ${JSON.stringify(dataStreamsContext)}.`) + if (dataStreamsContext) storage.enterWith({ ...(storage.getStore()), dataStreamsContext }) } diff --git a/packages/dd-trace/src/datastreams/pathway.js b/packages/dd-trace/src/datastreams/pathway.js index 5d587a4768f..066af789e64 100644 --- a/packages/dd-trace/src/datastreams/pathway.js +++ b/packages/dd-trace/src/datastreams/pathway.js @@ -4,6 +4,8 @@ const crypto = require('crypto') const { encodeVarint, decodeVarint } = require('./encoding') const LRUCache = require('lru-cache') +const log = require('../log') +const pick = require('../../../datadog-core/src/utils/src/pick') const options = { max: 500 } const cache = new LRUCache(options) @@ -11,6 +13,8 @@ const cache = new LRUCache(options) const CONTEXT_PROPAGATION_KEY = 'dd-pathway-ctx' const CONTEXT_PROPAGATION_KEY_BASE64 = 'dd-pathway-ctx-base64' +const logKeys = [CONTEXT_PROPAGATION_KEY, CONTEXT_PROPAGATION_KEY_BASE64] + function shaHash (checkpointString) { const hash = crypto.createHash('md5').update(checkpointString).digest('hex').slice(0, 16) return Buffer.from(hash, 'hex') @@ -80,9 +84,13 @@ class DsmPathwayCodec { return } carrier[CONTEXT_PROPAGATION_KEY_BASE64] = encodePathwayContextBase64(dataStreamsContext) + + log.debug(() => `Injected into DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) } static decode (carrier) { + log.debug(() => `Attempting extract from DSM carrier: ${JSON.stringify(pick(carrier, logKeys))}.`) + if (carrier == null) return let ctx @@ -97,13 +105,12 @@ class DsmPathwayCodec { // pass } // cover case where base64 context was received under wrong key - if (!ctx) ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY]) + if (!ctx && CONTEXT_PROPAGATION_KEY in carrier) { + ctx = decodePathwayContextBase64(carrier[CONTEXT_PROPAGATION_KEY]) + } } - return ctx - } - static contextExists (carrier) { - return CONTEXT_PROPAGATION_KEY_BASE64 in carrier || CONTEXT_PROPAGATION_KEY in carrier + return ctx } } diff --git a/packages/dd-trace/src/datastreams/processor.js b/packages/dd-trace/src/datastreams/processor.js index 8670c1571f5..d036af805a7 100644 --- a/packages/dd-trace/src/datastreams/processor.js +++ b/packages/dd-trace/src/datastreams/processor.js @@ -11,6 +11,7 @@ const { types } = require('util') const { PATHWAY_HASH } = require('../../../../ext/tags') const { SchemaBuilder } = require('./schemas/schema_builder') const { SchemaSampler } = require('./schemas/schema_sampler') +const log = require('../log') const ENTRY_PARENT_HASH = Buffer.from('0000000000000000', 'hex') @@ -272,6 +273,11 @@ class DataStreamsProcessor { closestOppositeDirectionHash = parentHash closestOppositeDirectionEdgeStart = edgeStartNs } + log.debug( + () => `Setting DSM Checkpoint from extracted parent context with hash: ${parentHash} and edge tags: ${edgeTags}` + ) + } else { + log.debug(() => 'Setting DSM Checkpoint with empty parent context.') } const hash = computePathwayHash(this.service, this.env, edgeTags, parentHash) const edgeLatencyNs = nowNs - edgeStartNs From 29c42f144466bc929beedda8c9e7744195823918 Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Wed, 2 Oct 2024 10:38:04 +0200 Subject: [PATCH 05/27] Update workflows to fix release process (#4747) --- .github/workflows/rebase-release-proposal.yml | 3 +++ .github/workflows/release-latest.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/rebase-release-proposal.yml b/.github/workflows/rebase-release-proposal.yml index 01ae84222b3..3ec2f1022a8 100644 --- a/.github/workflows/rebase-release-proposal.yml +++ b/.github/workflows/rebase-release-proposal.yml @@ -14,6 +14,9 @@ on: jobs: check: runs-on: ubuntu-latest + permissions: + id-token: write + contents: write steps: - name: Checkout repository diff --git a/.github/workflows/release-latest.yml b/.github/workflows/release-latest.yml index 4549f8ac714..6fa92f3ee23 100644 --- a/.github/workflows/release-latest.yml +++ b/.github/workflows/release-latest.yml @@ -36,6 +36,9 @@ jobs: docs: runs-on: ubuntu-latest + permissions: + id-token: write + contents: write needs: ['publish'] steps: - uses: actions/checkout@v4 From 92515a65e37d079ff3b3ea79546bc9d02dba82b0 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 2 Oct 2024 12:13:37 +0200 Subject: [PATCH 06/27] [DI] Add stack trace to log probe results (#4727) --- integration-tests/debugger/index.spec.js | 16 ++++++++++++++++ .../src/debugger/devtools_client/index.js | 13 +++++++++++++ .../debugger/devtools_client/remote_config.js | 4 ++-- .../src/debugger/devtools_client/state.js | 14 ++++++++++---- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/index.spec.js index 241f57f722f..4f975e179f6 100644 --- a/integration-tests/debugger/index.spec.js +++ b/integration-tests/debugger/index.spec.js @@ -308,6 +308,22 @@ describe('Dynamic Instrumentation', function () { assert.isTrue(payload['debugger.snapshot'].timestamp > Date.now() - 1000 * 60) assert.isTrue(payload['debugger.snapshot'].timestamp <= Date.now()) + assert.isArray(payload['debugger.snapshot'].stack) + assert.isAbove(payload['debugger.snapshot'].stack.length, 0) + for (const frame of payload['debugger.snapshot'].stack) { + assert.isObject(frame) + assert.hasAllKeys(frame, ['fileName', 'function', 'lineNumber', 'columnNumber']) + assert.isString(frame.fileName) + assert.isString(frame.function) + assert.isAbove(frame.lineNumber, 0) + assert.isAbove(frame.columnNumber, 0) + } + const topFrame = payload['debugger.snapshot'].stack[0] + assert.match(topFrame.fileName, new RegExp(`${appFile}$`)) // path seems to be prefeixed with `/private` on Mac + assert.strictEqual(topFrame.function, 'handler') + assert.strictEqual(topFrame.lineNumber, probeLineNo) + assert.strictEqual(topFrame.columnNumber, 3) + done() }) diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js index f4789ea65a8..e1100f99ab7 100644 --- a/packages/dd-trace/src/debugger/devtools_client/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -4,6 +4,7 @@ const { randomUUID } = require('crypto') const { breakpoints } = require('./state') const session = require('./session') const send = require('./send') +const { getScriptUrlFromId } = require('./state') const { ackEmitting } = require('./status') const { parentThreadId } = require('./config') const log = require('../../log') @@ -35,6 +36,17 @@ session.on('Debugger.paused', async ({ params }) => { thread_name: threadName } + const stack = params.callFrames.map((frame) => { + let fileName = getScriptUrlFromId(frame.location.scriptId) + if (fileName.startsWith('file://')) fileName = fileName.substr(7) // TODO: This might not be required + return { + fileName, + function: frame.functionName, + lineNumber: frame.location.lineNumber + 1, // Beware! lineNumber is zero-indexed + columnNumber: frame.location.columnNumber + 1 // Beware! columnNumber is zero-indexed + } + }) + // TODO: Send multiple probes in one HTTP request as an array for (const probe of probes) { const snapshot = { @@ -45,6 +57,7 @@ session.on('Debugger.paused', async ({ params }) => { version: probe.version, location: probe.location }, + stack, language: 'javascript' } diff --git a/packages/dd-trace/src/debugger/devtools_client/remote_config.js b/packages/dd-trace/src/debugger/devtools_client/remote_config.js index 25ac070cc9f..b030b093728 100644 --- a/packages/dd-trace/src/debugger/devtools_client/remote_config.js +++ b/packages/dd-trace/src/debugger/devtools_client/remote_config.js @@ -1,7 +1,7 @@ 'use strict' const { workerData: { rcPort } } = require('node:worker_threads') -const { getScript, probes, breakpoints } = require('./state') +const { findScriptFromPartialPath, probes, breakpoints } = require('./state') const session = require('./session') const { ackReceived, ackInstalled, ackError } = require('./status') const log = require('../../log') @@ -120,7 +120,7 @@ async function addBreakpoint (probe) { // TODO: Inbetween `await session.post('Debugger.enable')` and here, the scripts are parsed and cached. // Maybe there's a race condition here or maybe we're guraenteed that `await session.post('Debugger.enable')` will // not continue untill all scripts have been parsed? - const script = getScript(file) + const script = findScriptFromPartialPath(file) if (!script) throw new Error(`No loaded script found for ${file} (probe: ${probe.id}, version: ${probe.version})`) const [path, scriptId] = script diff --git a/packages/dd-trace/src/debugger/devtools_client/state.js b/packages/dd-trace/src/debugger/devtools_client/state.js index 316841667fb..8be9c808369 100644 --- a/packages/dd-trace/src/debugger/devtools_client/state.js +++ b/packages/dd-trace/src/debugger/devtools_client/state.js @@ -2,7 +2,8 @@ const session = require('./session') -const scripts = [] +const scriptIds = [] +const scriptUrls = new Map() module.exports = { probes: new Map(), @@ -25,10 +26,14 @@ module.exports = { * @param {string} path * @returns {[string, string] | undefined} */ - getScript (path) { - return scripts + findScriptFromPartialPath (path) { + return scriptIds .filter(([url]) => url.endsWith(path)) .sort(([a], [b]) => a.length - b.length)[0] + }, + + getScriptUrlFromId (id) { + return scriptUrls.get(id) } } @@ -41,7 +46,8 @@ module.exports = { // - `` - Not sure what this is, but should just be ignored // TODO: Event fired for all files, every time debugger is enabled. So when we disable it, we need to reset the state session.on('Debugger.scriptParsed', ({ params }) => { + scriptUrls.set(params.scriptId, params.url) if (params.url.startsWith('file:')) { - scripts.push([params.url, params.scriptId]) + scriptIds.push([params.url, params.scriptId]) } }) From 4d2f5b86a0e996c76aee85ff695658e277aefd87 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 2 Oct 2024 12:14:25 +0200 Subject: [PATCH 07/27] Don't use deprecated url.parse function (#4743) --- .../dd-trace/src/exporters/common/request.js | 42 ++++--------------- .../common/url-to-http-options-polyfill.js | 31 ++++++++++++++ 2 files changed, 39 insertions(+), 34 deletions(-) create mode 100644 packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js diff --git a/packages/dd-trace/src/exporters/common/request.js b/packages/dd-trace/src/exporters/common/request.js index 6823119c0d8..ab8b697eef6 100644 --- a/packages/dd-trace/src/exporters/common/request.js +++ b/packages/dd-trace/src/exporters/common/request.js @@ -6,10 +6,9 @@ const { Readable } = require('stream') const http = require('http') const https = require('https') -// eslint-disable-next-line n/no-deprecated-api -const { parse: urlParse } = require('url') const zlib = require('zlib') +const { urlToHttpOptions } = require('./url-to-http-options-polyfill') const docker = require('./docker') const { httpAgent, httpsAgent } = require('./agents') const { storage } = require('../../../../datadog-core') @@ -20,39 +19,14 @@ const containerId = docker.id() let activeRequests = 0 -// TODO: Replace with `url.urlToHttpOptions` when supported by all versions -function urlToOptions (url) { - const agent = url.agent || http.globalAgent - const options = { - protocol: url.protocol || agent.protocol, - hostname: typeof url.hostname === 'string' && url.hostname.startsWith('[') - ? url.hostname.slice(1, -1) - : url.hostname || - url.host || - 'localhost', - hash: url.hash, - search: url.search, - pathname: url.pathname, - path: `${url.pathname || ''}${url.search || ''}`, - href: url.href - } - if (url.port !== '') { - options.port = Number(url.port) - } - if (url.username || url.password) { - options.auth = `${url.username}:${url.password}` - } - return options -} +function parseUrl (urlObjOrString) { + if (typeof urlObjOrString === 'object') return urlToHttpOptions(urlObjOrString) -function fromUrlString (urlString) { - const url = typeof urlToHttpOptions === 'function' - ? urlToOptions(new URL(urlString)) - : urlParse(urlString) + const url = urlToHttpOptions(new URL(urlObjOrString)) - // Add the 'hostname' back if we're using named pipes - if (url.protocol === 'unix:' && url.host === '.') { - const udsPath = urlString.replace(/^unix:/, '') + // Special handling if we're using named pipes on Windows + if (url.protocol === 'unix:' && url.hostname === '.') { + const udsPath = urlObjOrString.slice(5) url.path = udsPath url.pathname = udsPath } @@ -66,7 +40,7 @@ function request (data, options, callback) { } if (options.url) { - const url = typeof options.url === 'object' ? urlToOptions(options.url) : fromUrlString(options.url) + const url = parseUrl(options.url) if (url.protocol === 'unix:') { options.socketPath = url.pathname } else { diff --git a/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js b/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js new file mode 100644 index 00000000000..4ba6b337b08 --- /dev/null +++ b/packages/dd-trace/src/exporters/common/url-to-http-options-polyfill.js @@ -0,0 +1,31 @@ +'use strict' + +const { urlToHttpOptions } = require('url') + +// TODO: Remove `urlToHttpOptions` polyfill once we drop support for the older Cypress versions that uses a built-in +// version of Node.js doesn't include that function. +module.exports = { + urlToHttpOptions: urlToHttpOptions ?? function (url) { + const { hostname, pathname, port, username, password, search } = url + const options = { + __proto__: null, + ...url, // In case the url object was extended by the user. + protocol: url.protocol, + hostname: typeof hostname === 'string' && hostname.startsWith('[') + ? hostname.slice(1, -1) + : hostname, + hash: url.hash, + search, + pathname, + path: `${pathname || ''}${search || ''}`, + href: url.href + } + if (port !== '') { + options.port = Number(port) + } + if (username || password) { + options.auth = `${decodeURIComponent(username)}:${decodeURIComponent(password)}` + } + return options + } +} From 70d5591d9b6f2f84c5fdd4fbdbad10bbd2de89a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 2 Oct 2024 12:29:44 +0200 Subject: [PATCH 08/27] =?UTF-8?q?[test=20visibility]=C2=A0Read=20`pull=5Fr?= =?UTF-8?q?equest`=20and=20`pull=5Frequest=5Ftarget`=20event=20info=20from?= =?UTF-8?q?=20GHA=20(#4745)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/dd-trace/src/plugins/util/ci.js | 24 ++++++- packages/dd-trace/src/plugins/util/tags.js | 7 ++ .../util/fixtures/github_event_payload.json | 70 +++++++++++++++++++ .../github_event_payload_malformed.json | 1 + .../plugins/util/test-environment.spec.js | 46 +++++++++++- 5 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json create mode 100644 packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json diff --git a/packages/dd-trace/src/plugins/util/ci.js b/packages/dd-trace/src/plugins/util/ci.js index 35e58c5a94e..86bda260212 100644 --- a/packages/dd-trace/src/plugins/util/ci.js +++ b/packages/dd-trace/src/plugins/util/ci.js @@ -1,3 +1,4 @@ +const { readFileSync } = require('fs') const { GIT_BRANCH, GIT_COMMIT_SHA, @@ -6,6 +7,9 @@ const { GIT_COMMIT_AUTHOR_NAME, GIT_COMMIT_MESSAGE, GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_PULL_REQUEST_BASE_BRANCH, GIT_REPOSITORY_URL, CI_PIPELINE_ID, CI_PIPELINE_NAME, @@ -77,6 +81,13 @@ function resolveTilde (filePath) { return filePath } +function getGitHubEventPayload () { + if (!process.env.GITHUB_EVENT_PATH) { + return + } + return JSON.parse(readFileSync(process.env.GITHUB_EVENT_PATH, 'utf8')) +} + module.exports = { normalizeRef, getCIMetadata () { @@ -241,7 +252,8 @@ module.exports = { GITHUB_REPOSITORY, GITHUB_SERVER_URL, GITHUB_RUN_ATTEMPT, - GITHUB_JOB + GITHUB_JOB, + GITHUB_BASE_REF } = env const repositoryURL = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git` @@ -277,6 +289,16 @@ module.exports = { GITHUB_RUN_ATTEMPT }) } + if (GITHUB_BASE_REF) { // `pull_request` or `pull_request_target` event + tags[GIT_PULL_REQUEST_BASE_BRANCH] = GITHUB_BASE_REF + try { + const eventContent = getGitHubEventPayload() + tags[GIT_PULL_REQUEST_BASE_BRANCH_SHA] = eventContent.pull_request.base.sha + tags[GIT_COMMIT_HEAD_SHA] = eventContent.pull_request.head.sha + } catch (e) { + // ignore malformed event content + } + } } if (env.APPVEYOR) { diff --git a/packages/dd-trace/src/plugins/util/tags.js b/packages/dd-trace/src/plugins/util/tags.js index 15a795f4c1d..58709f0ceb7 100644 --- a/packages/dd-trace/src/plugins/util/tags.js +++ b/packages/dd-trace/src/plugins/util/tags.js @@ -9,6 +9,10 @@ const GIT_COMMIT_COMMITTER_NAME = 'git.commit.committer.name' const GIT_COMMIT_AUTHOR_DATE = 'git.commit.author.date' const GIT_COMMIT_AUTHOR_EMAIL = 'git.commit.author.email' const GIT_COMMIT_AUTHOR_NAME = 'git.commit.author.name' +const GIT_COMMIT_HEAD_SHA = 'git.commit.head_sha' + +const GIT_PULL_REQUEST_BASE_BRANCH_SHA = 'git.pull_request.base_branch_sha' +const GIT_PULL_REQUEST_BASE_BRANCH = 'git.pull_request.base_branch' const CI_PIPELINE_ID = 'ci.pipeline.id' const CI_PIPELINE_NAME = 'ci.pipeline.name' @@ -36,6 +40,9 @@ module.exports = { GIT_COMMIT_AUTHOR_DATE, GIT_COMMIT_AUTHOR_EMAIL, GIT_COMMIT_AUTHOR_NAME, + GIT_COMMIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_PULL_REQUEST_BASE_BRANCH, CI_PIPELINE_ID, CI_PIPELINE_NAME, CI_PIPELINE_NUMBER, diff --git a/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json new file mode 100644 index 00000000000..64828fe2b7b --- /dev/null +++ b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload.json @@ -0,0 +1,70 @@ +{ + "action": "synchronize", + "after": "df289512a51123083a8e6931dd6f57bb3883d4c4", + "before": "f659d2fdd7bedffb40d9ab223dbde6afa5eadc32", + "number": 1, + "pull_request": { + "_links": {}, + "active_lock_reason": null, + "additions": 2, + "assignee": null, + "assignees": [], + "author_association": "OWNER", + "auto_merge": null, + "base": { + "label": "datadog:main", + "ref": "main", + "repo": {}, + "sha": "52e0974c74d41160a03d59ddc73bb9f5adab054b", + "user": {} + }, + "body": "# What Does This Do\r\n\r\n# Motivation\r\n\r\n# Additional Notes\r\n", + "changed_files": 3, + "closed_at": null, + "comments": 0, + "comments_url": "", + "commits": 2, + "commits_url": "", + "created_at": "2024-09-11T15:08:02Z", + "deletions": 0, + "diff_url": "", + "draft": false, + "head": { + "label": "forked_org:test-branch", + "ref": "test-branch", + "repo": {}, + "sha": "df289512a51123083a8e6931dd6f57bb3883d4c4", + "user": {} + }, + "html_url": "", + "id": 2066570986, + "issue_url": "", + "labels": [], + "locked": false, + "maintainer_can_modify": false, + "merge_commit_sha": "d9a3212d0d5d1483426dbbdf0beea32ee50abcde", + "mergeable": null, + "mergeable_state": "unknown", + "merged": false, + "merged_at": null, + "merged_by": null, + "milestone": null, + "node_id": "PR_kwDOIvpGAs57LV7q", + "number": 1, + "patch_url": "", + "rebaseable": null, + "requested_reviewers": [], + "requested_teams": [], + "review_comment_url": "", + "review_comments": 0, + "review_comments_url": "", + "state": "open", + "statuses_url": "", + "title": "Test commit", + "updated_at": "2024-09-11T15:12:26Z", + "url": "", + "user": {} + }, + "repository": {}, + "sender": {} +} diff --git a/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json new file mode 100644 index 00000000000..0967ef424bc --- /dev/null +++ b/packages/dd-trace/test/plugins/util/fixtures/github_event_payload_malformed.json @@ -0,0 +1 @@ +{} diff --git a/packages/dd-trace/test/plugins/util/test-environment.spec.js b/packages/dd-trace/test/plugins/util/test-environment.spec.js index 01f9b4914a4..63726622fb5 100644 --- a/packages/dd-trace/test/plugins/util/test-environment.spec.js +++ b/packages/dd-trace/test/plugins/util/test-environment.spec.js @@ -9,7 +9,13 @@ const proxyquire = require('proxyquire') const execFileSyncStub = sinon.stub().returns('') const { getCIMetadata } = require('../../../src/plugins/util/ci') -const { CI_ENV_VARS, CI_NODE_LABELS } = require('../../../src/plugins/util/tags') +const { + CI_ENV_VARS, + CI_NODE_LABELS, + GIT_PULL_REQUEST_BASE_BRANCH, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_COMMIT_HEAD_SHA +} = require('../../../src/plugins/util/tags') const { getGitMetadata } = proxyquire('../../../src/plugins/util/git', { child_process: { @@ -36,6 +42,44 @@ describe('test environment data', () => { const ciProviders = fs.readdirSync(path.join(__dirname, 'ci-env')) ciProviders.forEach(ciProvider => { const assertions = require(path.join(__dirname, 'ci-env', ciProvider)) + if (ciProvider === 'github.json') { + // We grab the first assertion because we only need to test one + const [env] = assertions[0] + it('can read pull request data from GitHub Actions', () => { + process.env = env + process.env.GITHUB_BASE_REF = 'datadog:main' + process.env.GITHUB_EVENT_PATH = path.join(__dirname, 'fixtures', 'github_event_payload.json') + const { + [GIT_PULL_REQUEST_BASE_BRANCH]: pullRequestBaseBranch, + [GIT_PULL_REQUEST_BASE_BRANCH_SHA]: pullRequestBaseBranchSha, + [GIT_COMMIT_HEAD_SHA]: headCommitSha + } = getTestEnvironmentMetadata() + + expect({ + pullRequestBaseBranch, + pullRequestBaseBranchSha, + headCommitSha + }).to.eql({ + pullRequestBaseBranch: 'datadog:main', + pullRequestBaseBranchSha: '52e0974c74d41160a03d59ddc73bb9f5adab054b', + headCommitSha: 'df289512a51123083a8e6931dd6f57bb3883d4c4' + }) + }) + it('does not crash if GITHUB_EVENT_PATH is not a valid JSON file', () => { + process.env = env + process.env.GITHUB_BASE_REF = 'datadog:main' + process.env.GITHUB_EVENT_PATH = path.join(__dirname, 'fixtures', 'github_event_payload_malformed.json') + const { + [GIT_PULL_REQUEST_BASE_BRANCH]: pullRequestBaseBranch, + [GIT_PULL_REQUEST_BASE_BRANCH_SHA]: pullRequestBaseBranchSha, + [GIT_COMMIT_HEAD_SHA]: headCommitSha + } = getTestEnvironmentMetadata() + + expect(pullRequestBaseBranch).to.equal('datadog:main') + expect(pullRequestBaseBranchSha).to.be.undefined + expect(headCommitSha).to.be.undefined + }) + } assertions.forEach(([env, expectedSpanTags], index) => { it(`reads env info for spec ${index} from ${ciProvider}`, () => { From 748ef616c37c2a399e81dac5f79fc430056aa549 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 2 Oct 2024 12:46:35 +0200 Subject: [PATCH 09/27] [DI] Switch unit tests to Mocha instead of Tap (#4728) --- package.json | 4 ++-- .../test/debugger/devtools_client/status.spec.js | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 75f97975880..821ed481d9a 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,8 @@ "test:appsec:ci": "nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" --exclude \"packages/dd-trace/test/appsec/**/*.plugin.spec.js\" -- npm run test:appsec", "test:appsec:plugins": "mocha -r \"packages/dd-trace/test/setup/mocha.js\" \"packages/dd-trace/test/appsec/**/*.@($(echo $PLUGINS)).plugin.spec.js\"", "test:appsec:plugins:ci": "yarn services && nyc --no-clean --include \"packages/dd-trace/src/appsec/**/*.js\" -- npm run test:appsec:plugins", - "test:debugger": "tap packages/dd-trace/test/debugger/**/*.spec.js", - "test:debugger:ci": "npm run test:debugger -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/debugger/**/*.js\"", + "test:debugger": "mocha -r 'packages/dd-trace/test/setup/mocha.js' 'packages/dd-trace/test/debugger/**/*.spec.js'", + "test:debugger:ci": "nyc --no-clean --include 'packages/dd-trace/src/debugger/**/*.js' -- npm run test:debugger", "test:trace:core": "tap packages/dd-trace/test/*.spec.js \"packages/dd-trace/test/{ci-visibility,datastreams,encode,exporters,opentelemetry,opentracing,plugins,service-naming,telemetry}/**/*.spec.js\"", "test:trace:core:ci": "npm run test:trace:core -- --coverage --nyc-arg=--include=\"packages/dd-trace/src/**/*.js\"", "test:instrumentations": "mocha -r 'packages/dd-trace/test/setup/mocha.js' 'packages/datadog-instrumentations/test/**/*.spec.js'", diff --git a/packages/dd-trace/test/debugger/devtools_client/status.spec.js b/packages/dd-trace/test/debugger/devtools_client/status.spec.js index 728279c7eca..41433f453c5 100644 --- a/packages/dd-trace/test/debugger/devtools_client/status.spec.js +++ b/packages/dd-trace/test/debugger/devtools_client/status.spec.js @@ -1,12 +1,12 @@ 'use strict' -require('../../setup/tap') +require('../../setup/mocha') const ddsource = 'dd_debugger' const service = 'my-service' const runtimeId = 'my-runtime-id' -describe('diagnostic message http request caching', () => { +describe('diagnostic message http request caching', function () { let statusproxy, request const acks = [ @@ -16,7 +16,7 @@ describe('diagnostic message http request caching', () => { ['ackError', 'ERROR', new Error('boom')] ] - beforeEach(() => { + beforeEach(function () { request = sinon.spy() request['@noCallThru'] = true @@ -27,10 +27,10 @@ describe('diagnostic message http request caching', () => { }) for (const [ackFnName, status, err] of acks) { - describe(ackFnName, () => { + describe(ackFnName, function () { let ackFn, exception - beforeEach(() => { + beforeEach(function () { if (err) { ackFn = statusproxy[ackFnName].bind(null, err) // Use `JSON.stringify` to remove any fields that are `undefined` @@ -45,7 +45,7 @@ describe('diagnostic message http request caching', () => { } }) - it('should only call once if no change', () => { + it('should only call once if no change', function () { ackFn({ id: 'foo', version: 0 }) expect(request).to.have.been.calledOnce assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) @@ -54,7 +54,7 @@ describe('diagnostic message http request caching', () => { expect(request).to.have.been.calledOnce }) - it('should call again if version changes', () => { + it('should call again if version changes', function () { ackFn({ id: 'foo', version: 0 }) expect(request).to.have.been.calledOnce assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) @@ -64,7 +64,7 @@ describe('diagnostic message http request caching', () => { assertRequestData(request, { probeId: 'foo', version: 1, status, exception }) }) - it('should call again if probeId changes', () => { + it('should call again if probeId changes', function () { ackFn({ id: 'foo', version: 0 }) expect(request).to.have.been.calledOnce assertRequestData(request, { probeId: 'foo', version: 0, status, exception }) From e09305d366bea4548547f01b10883da65e79ea90 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Wed, 2 Oct 2024 14:15:19 +0200 Subject: [PATCH 10/27] [DI] Fix probe.location.lines to be string[] instead of number[] (#4750) --- integration-tests/debugger/index.spec.js | 2 +- packages/dd-trace/src/debugger/devtools_client/remote_config.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/index.spec.js index 4f975e179f6..d0404393757 100644 --- a/integration-tests/debugger/index.spec.js +++ b/integration-tests/debugger/index.spec.js @@ -295,7 +295,7 @@ describe('Dynamic Instrumentation', function () { probe: { id: rcConfig.config.id, version: 0, - location: { file: probeFile, lines: [probeLineNo] } + location: { file: probeFile, lines: [String(probeLineNo)] } }, language: 'javascript' } diff --git a/packages/dd-trace/src/debugger/devtools_client/remote_config.js b/packages/dd-trace/src/debugger/devtools_client/remote_config.js index b030b093728..50d6976ef82 100644 --- a/packages/dd-trace/src/debugger/devtools_client/remote_config.js +++ b/packages/dd-trace/src/debugger/devtools_client/remote_config.js @@ -114,7 +114,7 @@ async function addBreakpoint (probe) { const line = Number(probe.where.lines[0]) // Tracer doesn't support multiple-line breakpoints // Optimize for sending data to /debugger/v1/input endpoint - probe.location = { file, lines: [line] } + probe.location = { file, lines: [String(line)] } delete probe.where // TODO: Inbetween `await session.post('Debugger.enable')` and here, the scripts are parsed and cached. From f988e003bf9849bb08aba7653fd872deee6f8021 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Thu, 3 Oct 2024 13:14:40 +0200 Subject: [PATCH 11/27] [DI] Add GitHub repo and SHA tags to probe results (#4751) --- .../dd-trace/src/debugger/devtools_client/config.js | 2 ++ .../dd-trace/src/debugger/devtools_client/send.js | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/dd-trace/src/debugger/devtools_client/config.js b/packages/dd-trace/src/debugger/devtools_client/config.js index 3e7c19715e1..838a1a76cca 100644 --- a/packages/dd-trace/src/debugger/devtools_client/config.js +++ b/packages/dd-trace/src/debugger/devtools_client/config.js @@ -7,6 +7,8 @@ const log = require('../../log') const config = module.exports = { runtimeId: parentConfig.tags['runtime-id'], service: parentConfig.service, + commitSHA: parentConfig.commitSHA, + repositoryUrl: parentConfig.repositoryUrl, parentThreadId } diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index 709e14d52b7..354df7c231c 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -1,18 +1,28 @@ 'use strict' +const { stringify } = require('querystring') + const config = require('./config') const request = require('../../exporters/common/request') +const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags') module.exports = send const ddsource = 'dd_debugger' const service = config.service +const ddtags = [ + [GIT_COMMIT_SHA, config.commitSHA], + [GIT_REPOSITORY_URL, config.repositoryUrl] +].map((pair) => pair.join(':')).join(',') + +const path = `/debugger/v1/input?${stringify({ ddtags })}` + function send (message, logger, snapshot, cb) { const opts = { method: 'POST', url: config.url, - path: '/debugger/v1/input', + path, headers: { 'Content-Type': 'application/json; charset=utf-8' } } From c700341689f44b5991d36c68967a21151255866d Mon Sep 17 00:00:00 2001 From: William Conti <58711692+wconti27@users.noreply.github.com> Date: Thu, 3 Oct 2024 09:40:25 -0400 Subject: [PATCH 12/27] prefix system-tests env var names (#4746) --- .github/workflows/system-tests.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index e1ce6f7d767..c53c5b3064c 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -47,10 +47,8 @@ jobs: TEST_LIBRARY: nodejs WEBLOG_VARIANT: ${{ matrix.weblog-variant }} DD_API_KEY: ${{ secrets.DD_API_KEY }} - AWS_ACCESS_KEY_ID: ${{ secrets.IDM_AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.IDM_AWS_SECRET_ACCESS_KEY }} - AWS_REGION: us-east-1 - AWS_DEFAULT_REGION: us-east-1 # AWS services should use `AWS_REGION`, but some still use the older `AWS_DEFAULT_REGION` + SYSTEM_TESTS_AWS_ACCESS_KEY_ID: ${{ secrets.IDM_AWS_ACCESS_KEY_ID }} + SYSTEM_TESTS_AWS_SECRET_ACCESS_KEY: ${{ secrets.IDM_AWS_SECRET_ACCESS_KEY }} steps: - name: Checkout system tests From eef6711411db5090e11620ef9bdc103908a2ee92 Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Fri, 4 Oct 2024 09:10:19 +0200 Subject: [PATCH 13/27] Fix child process not maintaining previous parent span after execution (#4752) * Fix child process not maintaining previous parent span after execution * adding some tests @bengl had written https://github.com/DataDog/dd-trace-js/pull/4540 --------- Co-authored-by: Thomas Hunter II --- .../src/child_process.js | 25 ++-- .../test/index.spec.js | 120 +++++++++++++++++- 2 files changed, 130 insertions(+), 15 deletions(-) diff --git a/packages/datadog-instrumentations/src/child_process.js b/packages/datadog-instrumentations/src/child_process.js index d8f56f90981..8af49788007 100644 --- a/packages/datadog-instrumentations/src/child_process.js +++ b/packages/datadog-instrumentations/src/child_process.js @@ -61,14 +61,17 @@ function wrapChildProcessSyncMethod (shell = false) { const childProcessInfo = normalizeArgs(arguments, shell) - return childProcessChannel.traceSync( - childProcessMethod, - { - command: childProcessInfo.command, - shell: childProcessInfo.shell - }, - this, - ...arguments) + const innerResource = new AsyncResource('bound-anonymous-fn') + return innerResource.runInAsyncScope(() => { + return childProcessChannel.traceSync( + childProcessMethod, + { + command: childProcessInfo.command, + shell: childProcessInfo.shell + }, + this, + ...arguments) + }) } } } @@ -101,6 +104,12 @@ function wrapChildProcessAsyncMethod (shell = false) { const childProcessInfo = normalizeArgs(arguments, shell) + const cb = arguments[arguments.length - 1] + if (typeof cb === 'function') { + const callbackResource = new AsyncResource('bound-anonymous-fn') + arguments[arguments.length - 1] = callbackResource.bind(cb) + } + const innerResource = new AsyncResource('bound-anonymous-fn') return innerResource.runInAsyncScope(() => { childProcessChannel.start.publish({ command: childProcessInfo.command, shell: childProcessInfo.shell }) diff --git a/packages/datadog-plugin-child_process/test/index.spec.js b/packages/datadog-plugin-child_process/test/index.spec.js index 4598457274e..33624eab4d8 100644 --- a/packages/datadog-plugin-child_process/test/index.spec.js +++ b/packages/datadog-plugin-child_process/test/index.spec.js @@ -283,6 +283,82 @@ describe('Child process plugin', () => { }) }) + describe('context maintenance', () => { + let parent + let childProcess + let tracer + + before(() => { + return agent.load(['child_process']) + .then(() => { + childProcess = require('child_process') + tracer = require('../../dd-trace') + tracer.init() + parent = tracer.startSpan('parent') + parent.finish() + }).then(_port => { + return new Promise(resolve => setImmediate(resolve)) + }) + }) + + after(() => { + return agent.close() + }) + + it('should preserve context around execSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around exec calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.exec('ls', () => { + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + + it('should preserve context around execFileSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execFileSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around execFile calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.execFile('ls', () => { + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + + it('should preserve context around spawnSync calls', () => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.spawnSync('ls') + expect(tracer.scope().active()).to.equal(parent) + }) + }) + + it('should preserve context around spawn calls', (done) => { + tracer.scope().activate(parent, () => { + expect(tracer.scope().active()).to.equal(parent) + childProcess.spawn('ls') + expect(tracer.scope().active()).to.equal(parent) + done() + }) + }) + }) + describe('Integration', () => { describe('Methods which spawn a shell by default', () => { const execAsyncMethods = ['exec'] @@ -299,19 +375,25 @@ describe('Child process plugin', () => { afterEach(() => agent.close({ ritmReset: false })) const parentSpanList = [true, false] - parentSpanList.forEach(parentSpan => { - describe(`${parentSpan ? 'with' : 'without'} parent span`, () => { + parentSpanList.forEach(hasParentSpan => { + let parentSpan + + describe(`${hasParentSpan ? 'with' : 'without'} parent span`, () => { const methods = [ ...execAsyncMethods.map(methodName => ({ methodName, async: true })), ...execSyncMethods.map(methodName => ({ methodName, async: false })) ] - if (parentSpan) { - beforeEach((done) => { - const parentSpan = tracer.startSpan('parent') + + beforeEach((done) => { + if (hasParentSpan) { + parentSpan = tracer.startSpan('parent') parentSpan.finish() tracer.scope().activate(parentSpan, done) - }) - } + } else { + storage.enterWith({}) + done() + } + }) methods.forEach(({ methodName, async }) => { describe(methodName, () => { @@ -335,6 +417,30 @@ describe('Child process plugin', () => { } }) + it('should maintain previous span after the execution', (done) => { + const res = childProcess[methodName]('ls') + const span = storage.getStore()?.span + expect(span).to.be.equals(parentSpan) + if (async) { + res.on('close', () => { + expect(span).to.be.equals(parentSpan) + done() + }) + } else { + done() + } + }) + + if (async) { + it('should maintain previous span in the callback', (done) => { + childProcess[methodName]('ls', () => { + const span = storage.getStore()?.span + expect(span).to.be.equals(parentSpan) + done() + }) + }) + } + it('command should be scrubbed', (done) => { const expected = { type: 'system', From d1f29dba99ee25aa5c2a9f415d28d25fed59d76e Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Fri, 4 Oct 2024 09:29:39 +0200 Subject: [PATCH 14/27] Fix appsec rate limiter flaky test (#4754) --- packages/dd-trace/test/appsec/reporter.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index c683bdc59fe..6fabf747bcf 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -253,7 +253,7 @@ describe('reporter', () => { expect(Reporter.reportAttack('', params)).to.not.be.false expect(addTags.getCall(5).firstArg).to.have.property('manual.keep').that.equals('true') done() - }, 1e3) + }, 1020) }) it('should not overwrite origin tag', () => { From d1abcab7a1b818fd15c00a9c2aad30ad841ff937 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 4 Oct 2024 09:37:04 +0200 Subject: [PATCH 15/27] [DI] Add hostname to probe result (#4756) --- integration-tests/debugger/index.spec.js | 3 +++ packages/dd-trace/src/debugger/devtools_client/send.js | 3 +++ 2 files changed, 6 insertions(+) diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/index.spec.js index d0404393757..111d4a6c6cd 100644 --- a/integration-tests/debugger/index.spec.js +++ b/integration-tests/debugger/index.spec.js @@ -2,6 +2,8 @@ const path = require('path') const { randomUUID } = require('crypto') +const os = require('os') + const getPort = require('get-port') const Axios = require('axios') const { assert } = require('chai') @@ -283,6 +285,7 @@ describe('Dynamic Instrumentation', function () { agent.on('debugger-input', ({ payload }) => { const expected = { ddsource: 'dd_debugger', + hostname: os.hostname(), service: 'node', message: 'Hello World!', logger: { diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index 354df7c231c..593c3ea235d 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -1,5 +1,6 @@ 'use strict' +const { hostname: getHostname } = require('os') const { stringify } = require('querystring') const config = require('./config') @@ -9,6 +10,7 @@ const { GIT_COMMIT_SHA, GIT_REPOSITORY_URL } = require('../../plugins/util/tags' module.exports = send const ddsource = 'dd_debugger' +const hostname = getHostname() const service = config.service const ddtags = [ @@ -28,6 +30,7 @@ function send (message, logger, snapshot, cb) { const payload = { ddsource, + hostname, service, message, logger, From a00c9c83614b8fcfab89053170adb1302f90266d Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Fri, 4 Oct 2024 09:44:27 +0200 Subject: [PATCH 16/27] Sql injection Exploit Prevention implementation for mysql2 library (#4712) --- .../datadog-instrumentations/src/mysql2.js | 221 +++++- .../test/mysql2.spec.js | 718 ++++++++++++++++++ packages/dd-trace/src/appsec/channels.js | 1 + .../dd-trace/src/appsec/rasp/sql_injection.js | 28 +- .../appsec/index.sequelize.plugin.spec.js | 2 +- .../rasp/sql_injection.mysql2.plugin.spec.js | 229 ++++++ .../test/appsec/rasp/sql_injection.spec.js | 67 +- packages/dd-trace/test/plugins/externals.json | 10 + 8 files changed, 1269 insertions(+), 7 deletions(-) create mode 100644 packages/datadog-instrumentations/test/mysql2.spec.js create mode 100644 packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js diff --git a/packages/datadog-instrumentations/src/mysql2.js b/packages/datadog-instrumentations/src/mysql2.js index 0077b6b9dda..096eec0e80e 100644 --- a/packages/datadog-instrumentations/src/mysql2.js +++ b/packages/datadog-instrumentations/src/mysql2.js @@ -6,11 +6,14 @@ const { AsyncResource } = require('./helpers/instrument') const shimmer = require('../../datadog-shimmer') +const semver = require('semver') -addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connection => { +addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, (Connection, version) => { const startCh = channel('apm:mysql2:query:start') const finishCh = channel('apm:mysql2:query:finish') const errorCh = channel('apm:mysql2:query:error') + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const shouldEmitEndAfterQueryAbort = semver.intersects(version, '>=1.3.3') shimmer.wrap(Connection.prototype, 'addCommand', addCommand => function (cmd) { if (!startCh.hasSubscribers) return addCommand.apply(this, arguments) @@ -28,6 +31,76 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connec return asyncResource.bind(addCommand, this).apply(this, arguments) }) + shimmer.wrap(Connection.prototype, 'query', query => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return query.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const addCommand = this.addCommand + this.addCommand = function (cmd) { return cmd } + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.addCommand = addCommand + } + + cb = queryCommand.onResult + + process.nextTick(() => { + if (cb) { + cb(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + if (shouldEmitEndAfterQueryAbort) { + queryCommand.emit('end') + } + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(Connection.prototype, 'execute', execute => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return execute.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const addCommand = this.addCommand + this.addCommand = function (cmd) { return cmd } + + let result + try { + result = execute.apply(this, arguments) + } finally { + this.addCommand = addCommand + } + + result?.onResult(abortController.signal.reason) + + return result + } + + return execute.apply(this, arguments) + }) + return Connection function bindExecute (cmd, execute, asyncResource) { @@ -79,3 +152,149 @@ addHook({ name: 'mysql2', file: 'lib/connection.js', versions: ['>=1'] }, Connec }, cmd)) } }) + +addHook({ name: 'mysql2', file: 'lib/pool.js', versions: ['>=1'] }, (Pool, version) => { + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const shouldEmitEndAfterQueryAbort = semver.intersects(version, '>=1.3.3') + + shimmer.wrap(Pool.prototype, 'query', query => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return query.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const getConnection = this.getConnection + this.getConnection = function () {} + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.getConnection = getConnection + } + + process.nextTick(() => { + if (queryCommand.onResult) { + queryCommand.onResult(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + if (shouldEmitEndAfterQueryAbort) { + queryCommand.emit('end') + } + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(Pool.prototype, 'execute', execute => function (sql, values, cb) { + if (!startOuterQueryCh.hasSubscribers) return execute.apply(this, arguments) + + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + if (typeof values === 'function') { + cb = values + } + + process.nextTick(() => { + cb(abortController.signal.reason) + }) + return + } + + return execute.apply(this, arguments) + }) + + return Pool +}) + +// PoolNamespace.prototype.query does not exist in mysql2<2.3.0 +addHook({ name: 'mysql2', file: 'lib/pool_cluster.js', versions: ['>=2.3.0'] }, PoolCluster => { + const startOuterQueryCh = channel('datadog:mysql2:outerquery:start') + const wrappedPoolNamespaces = new WeakSet() + + shimmer.wrap(PoolCluster.prototype, 'of', of => function () { + const poolNamespace = of.apply(this, arguments) + + if (startOuterQueryCh.hasSubscribers && !wrappedPoolNamespaces.has(poolNamespace)) { + shimmer.wrap(poolNamespace, 'query', query => function (sql, values, cb) { + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return query.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + const getConnection = this.getConnection + this.getConnection = function () {} + + let queryCommand + try { + queryCommand = query.apply(this, arguments) + } finally { + this.getConnection = getConnection + } + + process.nextTick(() => { + if (queryCommand.onResult) { + queryCommand.onResult(abortController.signal.reason) + } else { + queryCommand.emit('error', abortController.signal.reason) + } + + queryCommand.emit('end') + }) + + return queryCommand + } + + return query.apply(this, arguments) + }) + + shimmer.wrap(poolNamespace, 'execute', execute => function (sql, values, cb) { + if (typeof sql === 'object') sql = sql?.sql + + if (!sql) return execute.apply(this, arguments) + + const abortController = new AbortController() + startOuterQueryCh.publish({ sql, abortController }) + + if (abortController.signal.aborted) { + if (typeof values === 'function') { + cb = values + } + + process.nextTick(() => { + cb(abortController.signal.reason) + }) + + return + } + + return execute.apply(this, arguments) + }) + + wrappedPoolNamespaces.add(poolNamespace) + } + + return poolNamespace + }) + + return PoolCluster +}) diff --git a/packages/datadog-instrumentations/test/mysql2.spec.js b/packages/datadog-instrumentations/test/mysql2.spec.js new file mode 100644 index 00000000000..89e35f2a1f7 --- /dev/null +++ b/packages/datadog-instrumentations/test/mysql2.spec.js @@ -0,0 +1,718 @@ +'use strict' + +const { channel } = require('../src/helpers/instrument') +const agent = require('../../dd-trace/test/plugins/agent') +const { assert } = require('chai') +const semver = require('semver') + +describe('mysql2 instrumentation', () => { + withVersions('mysql2', 'mysql2', version => { + function abort ({ sql, abortController }) { + assert.isString(sql) + const error = new Error('Test') + abortController.abort(error) + + if (!abortController.signal.reason) { + abortController.signal.reason = error + } + } + + function noop () {} + + const config = { + host: '127.0.0.1', + user: 'root', + database: 'db' + } + + const sql = 'SELECT 1' + let startCh, mysql2, shouldEmitEndAfterQueryAbort + let apmQueryStartChannel, apmQueryStart, mysql2Version + + before(() => { + startCh = channel('datadog:mysql2:outerquery:start') + return agent.load(['mysql2']) + }) + + before(() => { + const mysql2Require = require(`../../../versions/mysql2@${version}`) + mysql2Version = mysql2Require.version() + // in v1.3.3 CommandQuery started to emit 'end' after 'error' event + shouldEmitEndAfterQueryAbort = semver.intersects(mysql2Version, '>=1.3.3') + mysql2 = mysql2Require.get() + apmQueryStartChannel = channel('apm:mysql2:query:start') + }) + + beforeEach(() => { + apmQueryStart = sinon.stub() + apmQueryStartChannel.subscribe(apmQueryStart) + }) + + afterEach(() => { + if (startCh?.hasSubscribers) { + startCh.unsubscribe(abort) + startCh.unsubscribe(noop) + } + apmQueryStartChannel.unsubscribe(apmQueryStart) + }) + + describe('lib/connection.js', () => { + let connection + + beforeEach(() => { + connection = mysql2.createConnection(config) + + connection.connect() + }) + + afterEach((done) => { + connection.end(() => done()) + }) + + describe('Connection.prototype.query', () => { + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = connection.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + connection.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + connection.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const query = connection.query(sql) + + query.on('error', (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = connection.query(sql) + + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const query = connection.query(sql) + + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }, null, {}) + connection.query(query) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }, null, {}) + + connection.query(query) + }) + + it('should work without subscriptions', (done) => { + const query = mysql2.Connection.createQuery(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }, null, {}) + + connection.query(query) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + connection.query(query) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + + connection.query(query) + }) + + it('should work without subscriptions', (done) => { + const query = mysql2.Connection.createQuery(sql, null, null, {}) + query.on('error', (err) => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + + connection.query(query) + }) + }) + }) + }) + + describe('Connection.prototype.execute', () => { + describe('with the query in options', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const options = { sql } + const commandExecute = connection.execute(options, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + done() + }) + + assert.equal(commandExecute.sql, options.sql) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with sql as string', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + connection.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + done() + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + connection.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const options = { sql } + + connection.execute(options, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + + describe('lib/pool.js', () => { + let pool + + before(() => { + pool = mysql2.createPool(config) + }) + + describe('Pool.prototype.query', () => { + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query({ sql }) + query.on('error', err => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + const query = pool.query({ sql }) + + query.on('error', err => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('without callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const query = pool.query(sql) + query.on('error', err => { + assert.propertyVal(err, 'message', 'Test') + sinon.assert.notCalled(apmQueryStart) + if (!shouldEmitEndAfterQueryAbort) done() + }) + + query.on('end', () => done()) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + const query = pool.query(sql) + + query.on('error', err => done(err)) + query.on('end', () => { + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + + describe('Pool.prototype.execute', () => { + describe('with object as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + pool.execute({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('with string as query', () => { + describe('with callback', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + pool.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + pool.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + pool.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + }) + + describe('lib/pool_cluster.js', () => { + let poolCluster, connection + + before(function () { + if (!semver.satisfies(mysql2Version, '>=2.3.0')) this.skip() + poolCluster = mysql2.createPoolCluster() + poolCluster.add('clusterA', config) + }) + + beforeEach((done) => { + poolCluster.getConnection('clusterA', function (err, _connection) { + if (err) { + done(err) + return + } + + connection = _connection + + done() + }) + }) + + afterEach(() => { + connection?.release() + }) + + describe('PoolNamespace.prototype.query', () => { + describe('with string as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.query(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with object as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.query({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + + describe('PoolNamespace.prototype.execute', () => { + describe('with string as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.execute(sql, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + + describe('with object as query', () => { + it('should abort the query on abortController.abort()', (done) => { + startCh.subscribe(abort) + + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.propertyVal(err, 'message', 'Test') + + setTimeout(() => { + sinon.assert.notCalled(apmQueryStart) + done() + }, 100) + }) + }) + + it('should work without abortController.abort()', (done) => { + startCh.subscribe(noop) + + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + + it('should work without subscriptions', (done) => { + const namespace = poolCluster.of() + namespace.execute({ sql }, (err) => { + assert.isNull(err) + sinon.assert.called(apmQueryStart) + + done() + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/src/appsec/channels.js b/packages/dd-trace/src/appsec/channels.js index c098efd5538..a451b9ce145 100644 --- a/packages/dd-trace/src/appsec/channels.js +++ b/packages/dd-trace/src/appsec/channels.js @@ -24,5 +24,6 @@ module.exports = { setUncaughtExceptionCaptureCallbackStart: dc.channel('datadog:process:setUncaughtExceptionCaptureCallback:start'), pgQueryStart: dc.channel('apm:pg:query:start'), pgPoolQueryStart: dc.channel('datadog:pg:pool:query:start'), + mysql2OuterQueryStart: dc.channel('datadog:mysql2:outerquery:start'), wafRunFinished: dc.channel('datadog:waf:run:finish') } diff --git a/packages/dd-trace/src/appsec/rasp/sql_injection.js b/packages/dd-trace/src/appsec/rasp/sql_injection.js index b942dd82be5..d4a165d8615 100644 --- a/packages/dd-trace/src/appsec/rasp/sql_injection.js +++ b/packages/dd-trace/src/appsec/rasp/sql_injection.js @@ -1,12 +1,18 @@ 'use strict' -const { pgQueryStart, pgPoolQueryStart, wafRunFinished } = require('../channels') +const { + pgQueryStart, + pgPoolQueryStart, + wafRunFinished, + mysql2OuterQueryStart +} = require('../channels') const { storage } = require('../../../../datadog-core') const addresses = require('../addresses') const waf = require('../waf') const { RULE_TYPES, handleResult } = require('./utils') const DB_SYSTEM_POSTGRES = 'postgresql' +const DB_SYSTEM_MYSQL = 'mysql' const reqQueryMap = new WeakMap() // WeakMap> let config @@ -17,18 +23,32 @@ function enable (_config) { pgQueryStart.subscribe(analyzePgSqlInjection) pgPoolQueryStart.subscribe(analyzePgSqlInjection) wafRunFinished.subscribe(clearQuerySet) + + mysql2OuterQueryStart.subscribe(analyzeMysql2SqlInjection) } function disable () { if (pgQueryStart.hasSubscribers) pgQueryStart.unsubscribe(analyzePgSqlInjection) if (pgPoolQueryStart.hasSubscribers) pgPoolQueryStart.unsubscribe(analyzePgSqlInjection) if (wafRunFinished.hasSubscribers) wafRunFinished.unsubscribe(clearQuerySet) + if (mysql2OuterQueryStart.hasSubscribers) mysql2OuterQueryStart.unsubscribe(analyzeMysql2SqlInjection) +} + +function analyzeMysql2SqlInjection (ctx) { + const query = ctx.sql + if (!query) return + + analyzeSqlInjection(query, DB_SYSTEM_MYSQL, ctx.abortController) } function analyzePgSqlInjection (ctx) { const query = ctx.query?.text if (!query) return + analyzeSqlInjection(query, DB_SYSTEM_POSTGRES, ctx.abortController) +} + +function analyzeSqlInjection (query, dbSystem, abortController) { const store = storage.getStore() if (!store) return @@ -39,7 +59,7 @@ function analyzePgSqlInjection (ctx) { let executedQueries = reqQueryMap.get(req) if (executedQueries?.has(query)) return - // Do not waste time executing same query twice + // Do not waste time checking same query twice // This also will prevent double calls in pg.Pool internal queries if (!executedQueries) { executedQueries = new Set() @@ -49,12 +69,12 @@ function analyzePgSqlInjection (ctx) { const persistent = { [addresses.DB_STATEMENT]: query, - [addresses.DB_SYSTEM]: DB_SYSTEM_POSTGRES + [addresses.DB_SYSTEM]: dbSystem } const result = waf.run({ persistent }, req, RULE_TYPES.SQL_INJECTION) - handleResult(result, req, res, ctx.abortController, config) + handleResult(result, req, res, abortController, config) } function hasInputAddress (payload) { diff --git a/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js index 07013a570d2..d444b82ec5e 100644 --- a/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js +++ b/packages/dd-trace/test/appsec/index.sequelize.plugin.spec.js @@ -30,7 +30,7 @@ describe('sequelize', () => { // close agent after(() => { appsec.disable() - return agent.close() + return agent.close({ ritmReset: false }) }) // init database diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js new file mode 100644 index 00000000000..2fe74e9f262 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.mysql2.plugin.spec.js @@ -0,0 +1,229 @@ +'use strict' + +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const Config = require('../../../src/config') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +describe('RASP - sql_injection', () => { + withVersions('mysql2', 'express', expressVersion => { + withVersions('mysql2', 'mysql2', mysql2Version => { + describe('sql injection with mysql2', () => { + const connectionData = { + host: '127.0.0.1', + user: 'root', + database: 'db' + } + let server, axios, app, mysql2 + + before(() => { + return agent.load(['express', 'http', 'mysql2'], { client: false }) + }) + + before(done => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + mysql2 = require(`../../../../../versions/mysql2@${mysql2Version}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('Test using Connection', () => { + let connection + + beforeEach(() => { + connection = mysql2.createConnection(connectionData) + connection.connect() + }) + + afterEach((done) => { + connection.end(() => done()) + }) + + describe('query', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + connection.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + connection.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + + describe('execute', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + connection.execute('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + connection.execute(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + }) + + describe('Test using Pool', () => { + let pool + + beforeEach(() => { + pool = mysql2.createPool(connectionData) + }) + + describe('query', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + pool.query('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + pool.query(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + + describe('execute', () => { + it('Should not detect threat', async () => { + app = (req, res) => { + pool.execute('SELECT ' + req.query.param, (err) => { + if (err) { + res.statusCode = 500 + } + + res.end() + }) + } + + axios.get('/?param=1') + + await checkRaspExecutedAndNotThreat(agent) + }) + + it('Should block query with callback', async () => { + app = (req, res) => { + pool.execute(`SELECT * FROM users WHERE id='${req.query.param}'`, (err) => { + if (err?.name === 'DatadogRaspAbortError') { + res.statusCode = 500 + } + res.end() + }) + } + + try { + await axios.get('/?param=\' OR 1 = 1 --') + } catch (e) { + return await checkRaspExecutedAndHasThreat(agent, 'rasp-sqli-rule-id-2') + } + + assert.fail('Request should be blocked') + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js index 5467f7ef150..d713521e986 100644 --- a/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js +++ b/packages/dd-trace/test/appsec/rasp/sql_injection.spec.js @@ -1,6 +1,6 @@ 'use strict' -const { pgQueryStart } = require('../../../src/appsec/channels') +const { pgQueryStart, mysql2OuterQueryStart } = require('../../../src/appsec/channels') const addresses = require('../../../src/appsec/addresses') const proxyquire = require('proxyquire') @@ -113,4 +113,69 @@ describe('RASP - sql_injection', () => { sinon.assert.notCalled(waf.run) }) }) + + describe('analyzeMysql2SqlInjection', () => { + it('should analyze sql injection', () => { + const ctx = { + sql: 'SELECT 1' + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + mysql2OuterQueryStart.publish(ctx) + + const persistent = { + [addresses.DB_STATEMENT]: 'SELECT 1', + [addresses.DB_SYSTEM]: 'mysql' + } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'sql_injection') + }) + + it('should not analyze sql injection if rasp is disabled', () => { + sqli.disable() + + const ctx = { + sql: 'SELECT 1' + } + const req = {} + datadogCore.storage.getStore.returns({ req }) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no store', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns(undefined) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no req', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns({}) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should not analyze sql injection if no query', () => { + const ctx = { + sql: 'SELECT 1' + } + datadogCore.storage.getStore.returns({}) + + mysql2OuterQueryStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) }) diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index eddbe0f887c..e0216047fa4 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -95,6 +95,16 @@ "versions": ["5", ">=6"] } ], + "mysql2": [ + { + "name": "mysql2", + "versions": ["1.3.3"] + }, + { + "name": "express", + "versions": [">=4"] + } + ], "fastify": [ { "name": "fastify", From d0247775159c91318c3d3367422474ef769630f8 Mon Sep 17 00:00:00 2001 From: Thomas Watson Date: Fri, 4 Oct 2024 09:52:39 +0200 Subject: [PATCH 17/27] [DI] Add ability to take state snapshot feature (#4549) Take a "snapshot" of the variables that are in scope when a probe is hit (except the global scope, which intentionally have been omitted since it's too noisy): - For each variable in scope, we traverse objects and their properties up to `maxReferenceDepth` deep (default is 3 levels). - Strings are truncated to `maxLength` (default is 255 characters). - Binary data is converted to strings with appropriate escaping of non printable characters (the `maxLength` limit is also applied) Out of scope: - Information about `this` is not captured. - maxCollectionSize limit - maxFieldCount limit - Special handling for snapshots larger than 1MB (e.g. snapshot pruning or something simpler) - PII redaction --- integration-tests/debugger/index.spec.js | 169 ++++- .../debugger/target-app/index.js | 35 + .../src/debugger/devtools_client/index.js | 48 +- .../debugger/devtools_client/remote_config.js | 2 +- .../devtools_client/snapshot/collector.js | 153 +++++ .../devtools_client/snapshot/index.js | 30 + .../devtools_client/snapshot/processor.js | 241 +++++++ packages/dd-trace/test/.eslintrc.json | 6 +- .../devtools_client/_inspected_file.js | 158 +++++ .../debugger/devtools_client/snapshot.spec.js | 601 ++++++++++++++++++ 10 files changed, 1434 insertions(+), 9 deletions(-) create mode 100644 packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js create mode 100644 packages/dd-trace/src/debugger/devtools_client/snapshot/index.js create mode 100644 packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js create mode 100644 packages/dd-trace/test/debugger/devtools_client/_inspected_file.js create mode 100644 packages/dd-trace/test/debugger/devtools_client/snapshot.spec.js diff --git a/integration-tests/debugger/index.spec.js b/integration-tests/debugger/index.spec.js index 111d4a6c6cd..613c4eeb695 100644 --- a/integration-tests/debugger/index.spec.js +++ b/integration-tests/debugger/index.spec.js @@ -12,7 +12,7 @@ const { ACKNOWLEDGED, ERROR } = require('../../packages/dd-trace/src/appsec/remo const { version } = require('../../package.json') const probeFile = 'debugger/target-app/index.js' -const probeLineNo = 9 +const probeLineNo = 14 const pollInterval = 1 describe('Dynamic Instrumentation', function () { @@ -275,7 +275,7 @@ describe('Dynamic Instrumentation', function () { }) describe('input messages', function () { - it('should capture and send expected snapshot when a log line probe is triggered', function (done) { + it('should capture and send expected payload when a log line probe is triggered', function (done) { agent.on('debugger-diagnostics', ({ payload }) => { if (payload.debugger.diagnostics.status === 'INSTALLED') { axios.get('/foo') @@ -392,6 +392,171 @@ describe('Dynamic Instrumentation', function () { agent.addRemoteConfig(rcConfig) }) + + describe('with snapshot', () => { + beforeEach(() => { + // Trigger the breakpoint once probe is successfully installed + agent.on('debugger-diagnostics', ({ payload }) => { + if (payload.debugger.diagnostics.status === 'INSTALLED') { + axios.get('/foo') + } + }) + }) + + it('should capture a snapshot', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + assert.deepEqual(Object.keys(captures), ['lines']) + assert.deepEqual(Object.keys(captures.lines), [String(probeLineNo)]) + + const { locals } = captures.lines[probeLineNo] + const { request, fastify, getSomeData } = locals + delete locals.request + delete locals.fastify + delete locals.getSomeData + + // from block scope + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' } + ] + }, + obj: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + baz: { type: 'number', value: '42' }, + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + deep: { + type: 'Object', + fields: { nested: { type: 'Object', notCapturedReason: 'depth' } } + } + } + }, + bar: { type: 'boolean', value: 'true' } + } + }, + emptyObj: { type: 'Object', fields: {} }, + fn: { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'fn' } + } + }, + p: { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, + '[[PromiseResult]]': { type: 'undefined' } + } + } + }) + + // from local scope + // There's no reason to test the `request` object 100%, instead just check its fingerprint + assert.deepEqual(Object.keys(request), ['type', 'fields']) + assert.equal(request.type, 'Request') + assert.deepEqual(request.fields.id, { type: 'string', value: 'req-1' }) + assert.deepEqual(request.fields.params, { + type: 'NullObject', fields: { name: { type: 'string', value: 'foo' } } + }) + assert.deepEqual(request.fields.query, { type: 'Object', fields: {} }) + assert.deepEqual(request.fields.body, { type: 'undefined' }) + + // from closure scope + // There's no reason to test the `fastify` object 100%, instead just check its fingerprint + assert.deepEqual(Object.keys(fastify), ['type', 'fields']) + assert.equal(fastify.type, 'Object') + + assert.deepEqual(getSomeData, { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'getSomeData' } + } + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true })) + }) + + it('should respect maxReferenceDepth', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[probeLineNo] + delete locals.request + delete locals.fastify + delete locals.getSomeData + + assert.deepEqual(locals, { + nil: { type: 'null', isNull: true }, + undef: { type: 'undefined' }, + bool: { type: 'boolean', value: 'true' }, + num: { type: 'number', value: '42' }, + bigint: { type: 'bigint', value: '42' }, + str: { type: 'string', value: 'foo' }, + lstr: { + type: 'string', + // eslint-disable-next-line max-len + value: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor i', + truncated: true, + size: 445 + }, + sym: { type: 'symbol', value: 'Symbol(foo)' }, + regex: { type: 'RegExp', value: '/bar/i' }, + arr: { type: 'Array', notCapturedReason: 'depth' }, + obj: { type: 'Object', notCapturedReason: 'depth' }, + emptyObj: { type: 'Object', notCapturedReason: 'depth' }, + fn: { type: 'Function', notCapturedReason: 'depth' }, + p: { type: 'Promise', notCapturedReason: 'depth' } + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxReferenceDepth: 0 } })) + }) + + it('should respect maxLength', (done) => { + agent.on('debugger-input', ({ payload: { 'debugger.snapshot': { captures } } }) => { + const { locals } = captures.lines[probeLineNo] + + assert.deepEqual(locals.lstr, { + type: 'string', + value: 'Lorem ipsu', + truncated: true, + size: 445 + }) + + done() + }) + + agent.addRemoteConfig(generateRemoteConfig({ captureSnapshot: true, capture: { maxLength: 10 } })) + }) + }) }) describe('race conditions', () => { diff --git a/integration-tests/debugger/target-app/index.js b/integration-tests/debugger/target-app/index.js index d0e1b7fb6dd..dd7f5e6328a 100644 --- a/integration-tests/debugger/target-app/index.js +++ b/integration-tests/debugger/target-app/index.js @@ -5,10 +5,17 @@ const Fastify = require('fastify') const fastify = Fastify() +// Since line probes have hardcoded line numbers, we want to try and keep the line numbers from changing within the +// `handler` function below when making changes to this file. This is achieved by calling `getSomeData` and keeping all +// variable names on the same line as much as possible. fastify.get('/:name', function handler (request) { + // eslint-disable-next-line no-unused-vars + const { nil, undef, bool, num, bigint, str, lstr, sym, regex, arr, obj, emptyObj, fn, p } = getSomeData() return { hello: request.params.name } }) +// WARNING: Breakpoints present above this line - Any changes to the lines above might influence tests! + fastify.listen({ port: process.env.APP_PORT }, (err) => { if (err) { fastify.log.error(err) @@ -16,3 +23,31 @@ fastify.listen({ port: process.env.APP_PORT }, (err) => { } process.send({ port: process.env.APP_PORT }) }) + +function getSomeData () { + return { + nil: null, + undef: undefined, + bool: true, + num: 42, + bigint: 42n, + str: 'foo', + // eslint-disable-next-line max-len + lstr: 'Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.', + sym: Symbol('foo'), + regex: /bar/i, + arr: [1, 2, 3], + obj: { + foo: { + baz: 42, + nil: null, + undef: undefined, + deep: { nested: { obj: { that: { goes: { on: { forever: true } } } } } } + }, + bar: true + }, + emptyObj: {}, + fn: () => {}, + p: Promise.resolve() + } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/index.js b/packages/dd-trace/src/debugger/devtools_client/index.js index e1100f99ab7..aa19c14ef64 100644 --- a/packages/dd-trace/src/debugger/devtools_client/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/index.js @@ -3,9 +3,10 @@ const { randomUUID } = require('crypto') const { breakpoints } = require('./state') const session = require('./session') +const { getLocalStateForCallFrame } = require('./snapshot') const send = require('./send') const { getScriptUrlFromId } = require('./state') -const { ackEmitting } = require('./status') +const { ackEmitting, ackError } = require('./status') const { parentThreadId } = require('./config') const log = require('../../log') const { version } = require('../../../../../package.json') @@ -20,9 +21,33 @@ const threadName = parentThreadId === 0 ? 'MainThread' : `WorkerThread:${parentT session.on('Debugger.paused', async ({ params }) => { const start = process.hrtime.bigint() const timestamp = Date.now() - const probes = params.hitBreakpoints.map((id) => breakpoints.get(id)) + + let captureSnapshotForProbe = null + let maxReferenceDepth, maxLength + const probes = params.hitBreakpoints.map((id) => { + const probe = breakpoints.get(id) + if (probe.captureSnapshot) { + captureSnapshotForProbe = probe + maxReferenceDepth = highestOrUndefined(probe.capture.maxReferenceDepth, maxReferenceDepth) + maxLength = highestOrUndefined(probe.capture.maxLength, maxLength) + } + return probe + }) + + let processLocalState + if (captureSnapshotForProbe !== null) { + try { + // TODO: Create unique states for each affected probe based on that probes unique `capture` settings (DEBUG-2863) + processLocalState = await getLocalStateForCallFrame(params.callFrames[0], { maxReferenceDepth, maxLength }) + } catch (err) { + // TODO: This error is not tied to a specific probe, but to all probes with `captureSnapshot: true`. + // However, in 99,99% of cases, there will be just a single probe, so I guess this simplification is ok? + ackError(err, captureSnapshotForProbe) // TODO: Ok to continue after sending ackError? + } + } + await session.post('Debugger.resume') - const diff = process.hrtime.bigint() - start // TODO: Should this be recored as telemetry? + const diff = process.hrtime.bigint() - start // TODO: Recored as telemetry (DEBUG-2858) log.debug(`Finished processing breakpoints - main thread paused for: ${Number(diff) / 1000000} ms`) @@ -47,7 +72,7 @@ session.on('Debugger.paused', async ({ params }) => { } }) - // TODO: Send multiple probes in one HTTP request as an array + // TODO: Send multiple probes in one HTTP request as an array (DEBUG-2848) for (const probe of probes) { const snapshot = { id: randomUUID(), @@ -61,10 +86,23 @@ session.on('Debugger.paused', async ({ params }) => { language: 'javascript' } - // TODO: Process template + if (probe.captureSnapshot) { + const state = processLocalState() + if (state) { + snapshot.captures = { + lines: { [probe.location.lines[0]]: { locals: state } } + } + } + } + + // TODO: Process template (DEBUG-2628) send(probe.template, logger, snapshot, (err) => { if (err) log.error(err) else ackEmitting(probe) }) } }) + +function highestOrUndefined (num, max) { + return num === undefined ? max : Math.max(num, max ?? 0) +} diff --git a/packages/dd-trace/src/debugger/devtools_client/remote_config.js b/packages/dd-trace/src/debugger/devtools_client/remote_config.js index 50d6976ef82..8a7d7386e33 100644 --- a/packages/dd-trace/src/debugger/devtools_client/remote_config.js +++ b/packages/dd-trace/src/debugger/devtools_client/remote_config.js @@ -92,7 +92,7 @@ async function processMsg (action, probe) { await addBreakpoint(probe) break case 'modify': - // TODO: Can we modify in place? + // TODO: Modify existing probe instead of removing it (DEBUG-2817) await removeBreakpoint(probe) await addBreakpoint(probe) break diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js new file mode 100644 index 00000000000..0a8848ce5e5 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/collector.js @@ -0,0 +1,153 @@ +'use strict' + +const session = require('../session') + +const LEAF_SUBTYPES = new Set(['date', 'regexp']) +const ITERABLE_SUBTYPES = new Set(['map', 'set', 'weakmap', 'weakset']) + +module.exports = { + getRuntimeObject: getObject +} + +// TODO: Can we speed up thread pause time by calling mutiple Runtime.getProperties in parallel when possible? +// The most simple solution would be to swich from an async/await approach to a callback based approach, in which case +// each lookup will just finish in its own time and traverse the child nodes when the event loop allows it. +// Alternatively, use `Promise.all` or something like that, but the code would probably be more complex. + +async function getObject (objectId, maxDepth, depth = 0) { + const { result, privateProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + if (privateProperties) result.push(...privateProperties) + + return traverseGetPropertiesResult(result, maxDepth, depth) +} + +async function traverseGetPropertiesResult (props, maxDepth, depth) { + // TODO: Decide if we should filter out non-enumerable properties or not: + // props = props.filter((e) => e.enumerable) + + if (depth >= maxDepth) return props + + for (const prop of props) { + if (prop.value === undefined) continue + const { value: { type, objectId, subtype } } = prop + if (type === 'object') { + if (objectId === undefined) continue // if `subtype` is "null" + if (LEAF_SUBTYPES.has(subtype)) continue // don't waste time with these subtypes + prop.value.properties = await getObjectProperties(subtype, objectId, maxDepth, depth) + } else if (type === 'function') { + prop.value.properties = await getFunctionProperties(objectId, maxDepth, depth + 1) + } + } + + return props +} + +async function getObjectProperties (subtype, objectId, maxDepth, depth) { + if (ITERABLE_SUBTYPES.has(subtype)) { + return getIterable(objectId, maxDepth, depth) + } else if (subtype === 'promise') { + return getInternalProperties(objectId, maxDepth, depth) + } else if (subtype === 'proxy') { + return getProxy(objectId, maxDepth, depth) + } else if (subtype === 'arraybuffer') { + return getArrayBuffer(objectId, maxDepth, depth) + } else { + return getObject(objectId, maxDepth, depth + 1) + } +} + +// TODO: The following extra information from `internalProperties` might be relevant to include for functions: +// - Bound function: `[[TargetFunction]]`, `[[BoundThis]]` and `[[BoundArgs]]` +// - Non-bound function: `[[FunctionLocation]]`, and `[[Scopes]]` +async function getFunctionProperties (objectId, maxDepth, depth) { + let { result } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // For legacy reasons (I assume) functions has a `prototype` property besides the internal `[[Prototype]]` + result = result.filter(({ name }) => name !== 'prototype') + + return traverseGetPropertiesResult(result, maxDepth, depth) +} + +async function getIterable (objectId, maxDepth, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + let entry = internalProperties[1] + if (entry.name !== '[[Entries]]') { + // Currently `[[Entries]]` is the last of 2 elements, but in case this ever changes, fall back to searching + entry = internalProperties.findLast(({ name }) => name === '[[Entries]]') + } + + // Skip the `[[Entries]]` level and go directly to the content of the iterable + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + return traverseGetPropertiesResult(result, maxDepth, depth) +} + +async function getInternalProperties (objectId, maxDepth, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // We want all internal properties except the prototype + const props = internalProperties.filter(({ name }) => name !== '[[Prototype]]') + + return traverseGetPropertiesResult(props, maxDepth, depth) +} + +async function getProxy (objectId, maxDepth, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // TODO: If we do not skip the proxy wrapper, we can add a `revoked` boolean + let entry = internalProperties[1] + if (entry.name !== '[[Target]]') { + // Currently `[[Target]]` is the last of 2 elements, but in case this ever changes, fall back to searching + entry = internalProperties.findLast(({ name }) => name === '[[Target]]') + } + + // Skip the `[[Target]]` level and go directly to the target of the Proxy + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + return traverseGetPropertiesResult(result, maxDepth, depth) +} + +// Support for ArrayBuffer is a bit trickly because the internal structure stored in `internalProperties` is not +// documented and is not straight forward. E.g. ArrayBuffer(3) will internally contain both Int8Array(3) and +// UInt8Array(3), whereas ArrayBuffer(8) internally contains both Int8Array(8), Uint8Array(8), Int16Array(4), and +// Int32Array(2) - all representing the same data in different ways. +async function getArrayBuffer (objectId, maxDepth, depth) { + const { internalProperties } = await session.post('Runtime.getProperties', { + objectId, + ownProperties: true // exclude inherited properties + }) + + // Use Uint8 to make it easy to convert to a string later. + const entry = internalProperties.find(({ name }) => name === '[[Uint8Array]]') + + // Skip the `[[Uint8Array]]` level and go directly to the content of the ArrayBuffer + const { result } = await session.post('Runtime.getProperties', { + objectId: entry.value.objectId, + ownProperties: true // exclude inherited properties + }) + + return traverseGetPropertiesResult(result, maxDepth, depth) +} diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js new file mode 100644 index 00000000000..add097ac755 --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js @@ -0,0 +1,30 @@ +'use strict' + +const { getRuntimeObject } = require('./collector') +const { processRawState } = require('./processor') + +const DEFAULT_MAX_REFERENCE_DEPTH = 3 +const DEFAULT_MAX_LENGTH = 255 + +module.exports = { + getLocalStateForCallFrame +} + +async function getLocalStateForCallFrame ( + callFrame, + { maxReferenceDepth = DEFAULT_MAX_REFERENCE_DEPTH, maxLength = DEFAULT_MAX_LENGTH } = {} +) { + const rawState = [] + let processedState = null + + for (const scope of callFrame.scopeChain) { + if (scope.type === 'global') continue // The global scope is too noisy + rawState.push(...await getRuntimeObject(scope.object.objectId, maxReferenceDepth)) + } + + // Deplay calling `processRawState` so the caller gets a chance to resume the main thread before processing `rawState` + return () => { + processedState = processedState ?? processRawState(rawState, maxLength) + return processedState + } +} diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js new file mode 100644 index 00000000000..2cac9ef0b1c --- /dev/null +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/processor.js @@ -0,0 +1,241 @@ +'use strict' + +module.exports = { + processRawState: processProperties +} + +// Matches classes in source code, no matter how it's written: +// - Named: class MyClass {} +// - Anonymous: class {} +// - Named, with odd whitespace: class\n\t MyClass\n{} +// - Anonymous, with odd whitespace: class\n{} +const CLASS_REGEX = /^class\s([^{]*)/ + +function processProperties (props, maxLength) { + const result = {} + + for (const prop of props) { + // TODO: Hack to avoid periods in keys, as EVP doesn't support that. A better solution can be implemented later + result[prop.name.replaceAll('.', '_')] = getPropertyValue(prop, maxLength) + } + + return result +} + +function getPropertyValue (prop, maxLength) { + // Special case for getters and setters which does not have a value property + if ('get' in prop) { + const hasGet = prop.get.type !== 'undefined' + const hasSet = prop.set.type !== 'undefined' + if (hasGet && hasSet) return { type: 'getter/setter' } + if (hasGet) return { type: 'getter' } + if (hasSet) return { type: 'setter' } + } + + switch (prop.value?.type) { + case 'object': + return getObjectValue(prop.value, maxLength) + case 'function': + return toFunctionOrClass(prop.value, maxLength) + case undefined: // TODO: Add test for when a prop has no value. I think it's if it's defined after the breakpoint? + case 'undefined': + return { type: 'undefined' } + case 'string': + return toString(prop.value.value, maxLength) + case 'number': + return { type: 'number', value: prop.value.description } // use `descripton` to get it as string + case 'boolean': + return { type: 'boolean', value: prop.value.value === true ? 'true' : 'false' } + case 'symbol': + return { type: 'symbol', value: prop.value.description } + case 'bigint': + return { type: 'bigint', value: prop.value.description.slice(0, -1) } // remove trailing `n` + default: + // As of this writing, the Chrome DevTools Protocol doesn't allow any other types than the ones listed above, but + // in the future new ones might be added. + return { type: prop.value.type, notCapturedReason: 'Unsupported property type' } + } +} + +function getObjectValue (obj, maxLength) { + switch (obj.subtype) { + case undefined: + return toObject(obj.className, obj.properties, maxLength) + case 'array': + return toArray(obj.className, obj.properties, maxLength) + case 'null': + return { type: 'null', isNull: true } + // case 'node': // TODO: What does this subtype represent? + case 'regexp': + return { type: obj.className, value: obj.description } + case 'date': + // TODO: This looses millisecond resolution, as that's not retained in the `.toString()` representation contained + // in the `description` field. Unfortunately that's all we get from the Chrome DevTools Protocol. + return { type: obj.className, value: `${new Date(obj.description).toISOString().slice(0, -5)}Z` } + case 'map': + return toMap(obj.className, obj.properties, maxLength) + case 'set': + return toSet(obj.className, obj.properties, maxLength) + case 'weakmap': + return toMap(obj.className, obj.properties, maxLength) + case 'weakset': + return toSet(obj.className, obj.properties, maxLength) + // case 'iterator': // TODO: I've not been able to trigger this subtype + case 'generator': + // Use `subtype` instead of `className` to make it obvious it's a generator + return toObject(obj.subtype, obj.properties, maxLength) + case 'error': + // TODO: Convert stack trace to array to avoid string trunctation or disable truncation in this case? + return toObject(obj.className, obj.properties, maxLength) + case 'proxy': + // Use `desciption` instead of `className` as the `type` to get type of target object (`Proxy(Error)` vs `proxy`) + return toObject(obj.description, obj.properties, maxLength) + case 'promise': + return toObject(obj.className, obj.properties, maxLength) + case 'typedarray': + return toArray(obj.className, obj.properties, maxLength) + case 'arraybuffer': + return toArrayBuffer(obj.className, obj.properties, maxLength) + // case 'dataview': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter + // case 'webassemblymemory': // TODO: Looks like the internal ArrayBuffer is only accessible via the `buffer` getter + // case 'wasmvalue': // TODO: I've not been able to trigger this subtype + default: + // As of this writing, the Chrome DevTools Protocol doesn't allow any other subtypes than the ones listed above, + // but in the future new ones might be added. + return { type: obj.subtype, notCapturedReason: 'Unsupported object type' } + } +} + +function toFunctionOrClass (value, maxLength) { + const classMatch = value.description.match(CLASS_REGEX) + + if (classMatch === null) { + // This is a function + // TODO: Would it make sense to detect if it's an arrow function or not? + return toObject(value.className, value.properties, maxLength) + } else { + // This is a class + const className = classMatch[1].trim() + return { type: className ? `class ${className}` : 'class' } + } +} + +function toString (str, maxLength) { + const size = str.length + + if (size <= maxLength) { + return { type: 'string', value: str } + } + + return { + type: 'string', + value: str.substr(0, maxLength), + truncated: true, + size + } +} + +function toObject (type, props, maxLength) { + if (props === undefined) return notCapturedDepth(type) + return { type, fields: processProperties(props, maxLength) } +} + +function toArray (type, elements, maxLength) { + if (elements === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) + const expectedLength = elements.length - 1 + const result = { type, elements: new Array(expectedLength) } + + let i = 0 + for (const elm of elements) { + if (elm.enumerable === false) continue // the value of the `length` property should not be part of the array + result.elements[i++] = getPropertyValue(elm, maxLength) + } + + // Safe-guard in case there were more than one non-enumerable element + if (i < expectedLength) result.elements.length = i + + return result +} + +function toMap (type, pairs, maxLength) { + if (pairs === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) + const expectedLength = pairs.length - 1 + const result = { type, entries: new Array(expectedLength) } + + let i = 0 + for (const pair of pairs) { + if (pair.enumerable === false) continue // the value of the `length` property should not be part of the map + // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. + // There doesn't seem to be any documentation to back it up: + // + // `pair.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go + // directly to its children, of which there will always be exactly two, the first containing the key, and the + // second containing the value of this entry of the Map. + const key = getPropertyValue(pair.value.properties[0], maxLength) + const val = getPropertyValue(pair.value.properties[1], maxLength) + result.entries[i++] = [key, val] + } + + // Safe-guard in case there were more than one non-enumerable element + if (i < expectedLength) result.entries.length = i + + return result +} + +function toSet (type, values, maxLength) { + if (values === undefined) return notCapturedDepth(type) + + // Perf: Create array of expected size in advance (expect that it contains only one non-enumrable element) + const expectedLength = values.length - 1 + const result = { type, elements: new Array(expectedLength) } + + let i = 0 + for (const value of values) { + if (value.enumerable === false) continue // the value of the `length` property should not be part of the set + // The following code is based on assumptions made when researching the output of the Chrome DevTools Protocol. + // There doesn't seem to be any documentation to back it up: + // + // `value.value` is a special wrapper-object with subtype `internal#entry`. This can be skipped and we can go + // directly to its children, of which there will always be exactly one, which contain the actual value in this entry + // of the Set. + result.elements[i++] = getPropertyValue(value.value.properties[0], maxLength) + } + + // Safe-guard in case there were more than one non-enumerable element + if (i < expectedLength) result.elements.length = i + + return result +} + +function toArrayBuffer (type, bytes, maxLength) { + if (bytes === undefined) return notCapturedDepth(type) + + const size = bytes.length + + if (size > maxLength) { + return { + type, + value: arrayBufferToString(bytes, maxLength), + truncated: true, + size: bytes.length + } + } else { + return { type, value: arrayBufferToString(bytes, size) } + } +} + +function arrayBufferToString (bytes, size) { + const buf = Buffer.allocUnsafe(size) + for (let i = 0; i < size; i++) { + buf[i] = bytes[i].value.value + } + return buf.toString() +} + +function notCapturedDepth (type) { + return { type, notCapturedReason: 'depth' } +} diff --git a/packages/dd-trace/test/.eslintrc.json b/packages/dd-trace/test/.eslintrc.json index ed8a9ff7a87..3a9e197c393 100644 --- a/packages/dd-trace/test/.eslintrc.json +++ b/packages/dd-trace/test/.eslintrc.json @@ -2,8 +2,12 @@ "extends": [ "../../../.eslintrc.json" ], + "parserOptions": { + "ecmaVersion": 2022 + }, "env": { - "mocha": true + "mocha": true, + "es2022": true }, "globals": { "expect": true, diff --git a/packages/dd-trace/test/debugger/devtools_client/_inspected_file.js b/packages/dd-trace/test/debugger/devtools_client/_inspected_file.js new file mode 100644 index 00000000000..c7c27cd207b --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/_inspected_file.js @@ -0,0 +1,158 @@ +'use strict' + +function getPrimitives (a1 = 1, a2 = 2) { + // eslint-disable-next-line no-unused-vars + const { undef, nil, bool, num, bigint, str, sym } = get().primitives + return 'my return value' +} + +function getComplextTypes (a1 = 1, a2 = 2) { + // eslint-disable-next-line no-unused-vars, max-len + const { oblit, obnew, arr, regex, date, map, set, wmap, wset, gen, err, fn, bfn, afn, cls, acls, prox, custProx, pPen, pRes, pRej, tarr, ab, sab, circular, hidden } = get().complexTypes + return 'my return value' +} + +function getNestedObj (a1 = 1, a2 = 2) { + // eslint-disable-next-line no-unused-vars + const { myNestedObj } = get().nested + return 'my return value' +} + +// WARNING: Breakpoints present above this line - Any changes to the lines above might influence tests! + +// References to objects used in WeakMap/WeakSet objects to ensure that they are not garbage collected during testing +const ref = { + wmo1: { a: 1 }, + wmo2: { b: 3 }, + wso1: { a: 1 }, + wso2: { a: 2 }, + wso3: { a: 3 } +} + +// warp it all in a single function to avoid spamming the closure scope with a lot of variables (makes testing simpler) +function get () { + const bigint = BigInt(Number.MAX_SAFE_INTEGER) * 2n + + let e, g + const oblit = { + a: 1, + 'b.b': 2, + [Symbol('c')]: 3, + // Has no side-effect + // TODO: At some point it would be great to detect this and get the value, + // though currently we can neither detect it, nor execute the getter. + get d () { + return 4 + }, + // Has side-effect: We should never try to fetch this! + get e () { + e = Math.random() + return e + }, + // Only setter + set f (v) {}, // eslint-disable-line accessor-pairs + // Both getter and setter + get g () { return g }, + set g (x) { g = x } + } + + function fnWithProperties (a, b) {} + fnWithProperties.foo = { bar: 42 } + + class MyClass { + #secret = 42 + constructor () { + this.foo = this.#secret + } + } + + function * makeIterator () { + yield 1 + yield 2 + } + const gen = makeIterator() + gen.foo = 42 + + class CustomError extends Error { + constructor (...args) { + super(...args) + this.foo = 42 + } + } + const err = new CustomError('boom!') + + const buf1 = Buffer.from('IBM') + const buf2 = Buffer.from('hello\x01\x02\x03world') + + const arrayBuffer = new ArrayBuffer(buf1.length) + const sharedArrayBuffer = new SharedArrayBuffer(buf2.length) + + const typedArray = new Int8Array(arrayBuffer) + for (let i = 0; i < buf1.length; i++) typedArray[i] = buf1[i] - 1 + + const sharedTypedArray = new Int8Array(sharedArrayBuffer) + for (let i = 0; i < buf2.length; i++) sharedTypedArray[i] = buf2[i] + + const result = { + primitives: { + undef: undefined, + nil: null, + bool: true, + num: 42, + bigint, + str: 'foo', + sym: Symbol('foo') + }, + complexTypes: { + oblit, + obnew: new MyClass(), + arr: [1, 2, 3], + regex: /foo/, + date: new Date('2024-09-20T07:22:59.998Z'), + map: new Map([[1, 2], [3, 4]]), + set: new Set([[1, 2], 3, 4]), + wmap: new WeakMap([[ref.wmo1, 2], [ref.wmo2, 4]]), + wset: new WeakSet([ref.wso1, ref.wso2, ref.wso3]), + gen, + err, + fn: fnWithProperties, + bfn: fnWithProperties.bind(new MyClass(), 1, 2), + afn: () => { return 42 }, + cls: MyClass, + acls: class + {}, // eslint-disable-line indent, brace-style + prox: new Proxy({ target: true }, { get () { return false } }), + custProx: new Proxy(new MyClass(), { get () { return false } }), + pPen: new Promise(() => {}), + pRes: Promise.resolve('resolved value'), + pRej: Promise.reject('rejected value'), // eslint-disable-line prefer-promise-reject-errors + tarr: typedArray, // TODO: Should we test other TypedArray's? + ab: arrayBuffer, + sab: sharedArrayBuffer + }, + nested: { + myNestedObj: { + deepObj: { foo: { foo: { foo: { foo: { foo: true } } } } }, + deepArr: [[[[[42]]]]] + } + } + } + + result.complexTypes.circular = result.complexTypes + + Object.defineProperty(result.complexTypes, 'hidden', { + value: 'secret', + enumerable: false + }) + + // ensure we don't get an unhandled promise rejection error + result.complexTypes.pRej.catch(() => {}) + + return result +} + +module.exports = { + getPrimitives, + getComplextTypes, + getNestedObj +} diff --git a/packages/dd-trace/test/debugger/devtools_client/snapshot.spec.js b/packages/dd-trace/test/debugger/devtools_client/snapshot.spec.js new file mode 100644 index 00000000000..ce099ee00e3 --- /dev/null +++ b/packages/dd-trace/test/debugger/devtools_client/snapshot.spec.js @@ -0,0 +1,601 @@ +'use strict' + +require('../../setup/mocha') + +const NODE_20_PLUS = require('semver').gte(process.version, '20.0.0') + +const inspector = require('../../../src/debugger/devtools_client/inspector_promises_polyfill') +const session = new inspector.Session() +session.connect() + +session['@noCallThru'] = true +proxyquire('../src/debugger/devtools_client/snapshot/collector', { + '../session': session +}) + +const { getPrimitives, getComplextTypes, getNestedObj } = require('./_inspected_file') +const { getLocalStateForCallFrame } = require('../../../src/debugger/devtools_client/snapshot') + +let scriptId + +describe('debugger -> devtools client -> snapshot.getLocalStateForCallFrame', function () { + beforeEach(async function () { + scriptId = new Promise((resolve) => { + session.on('Debugger.scriptParsed', ({ params }) => { + if (params.url.endsWith('/_inspected_file.js')) { + session.removeAllListeners('Debugger.scriptParsed') // TODO: Can we do this in prod code? + resolve(params.scriptId) + } + }) + }) + + await session.post('Debugger.enable') + }) + + afterEach(async function () { + await session.post('Debugger.disable') + }) + + it('should return expected object for primitives', async function () { + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + const state = (await getLocalStateForCallFrame(params.callFrames[0]))() + + expect(Object.keys(state).length).to.equal(11) + + // from block scope + expect(state).to.have.deep.property('undef', { type: 'undefined' }) + expect(state).to.have.deep.property('nil', { type: 'null', isNull: true }) + expect(state).to.have.deep.property('bool', { type: 'boolean', value: 'true' }) + expect(state).to.have.deep.property('num', { type: 'number', value: '42' }) + expect(state).to.have.deep.property('bigint', { type: 'bigint', value: '18014398509481982' }) + expect(state).to.have.deep.property('str', { type: 'string', value: 'foo' }) + expect(state).to.have.deep.property('sym', { type: 'symbol', value: 'Symbol(foo)' }) + + // from local scope + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wmo2: { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + wso1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wso2: { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + wso3: { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + await setBreakpointOnLine(6) + getPrimitives() + }) + + describe('should return expected object for complex types', function () { + let state + + beforeEach(async function () { + let resolve + const localState = new Promise((_resolve) => { resolve = _resolve }) + + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + resolve((await getLocalStateForCallFrame(params.callFrames[0]))()) + }) + + await setBreakpointOnLine(12) + getComplextTypes() + + state = await localState + }) + + it('should contain expected properties from local and closure scope', function () { + expect(Object.keys(state).length).to.equal(30) + + // from block scope + // ... tested individually in the remaining it-blocks inside this describe-block + + // from local scope + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wmo2: { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + wso1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wso2: { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + wso3: { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + it('object literal', function () { + expect(state).to.have.deep.property('oblit', { + type: 'Object', + fields: { + a: { type: 'number', value: '1' }, + b_b: { type: 'number', value: '2' }, + 'Symbol(c)': { type: 'number', value: '3' }, + d: { type: 'getter' }, + e: { type: 'getter' }, + f: { type: 'setter' }, + g: { type: 'getter/setter' } + } + }) + }) + + it('custom object from class', function () { + expect(state).to.have.deep.property('obnew', { + type: 'MyClass', + fields: { + foo: { type: 'number', value: '42' }, + '#secret': { type: 'number', value: '42' } + } + }) + }) + + it('Array', function () { + expect(state).to.have.deep.property('arr', { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' }, + { type: 'number', value: '3' } + ] + }) + }) + + it('RegExp', function () { + expect(state).to.have.deep.property('regex', { type: 'RegExp', value: '/foo/' }) + }) + + it('Date', function () { + expect(state).to.have.deep.property('date', { + type: 'Date', + value: '2024-09-20T07:22:59Z' // missing milliseconds due to API limitation (should have been `998`) + }) + }) + + it('Map', function () { + expect(state).to.have.deep.property('map', { + type: 'Map', + entries: [ + [{ type: 'number', value: '1' }, { type: 'number', value: '2' }], + [{ type: 'number', value: '3' }, { type: 'number', value: '4' }] + ] + }) + }) + + it('Set', function () { + expect(state).to.have.deep.property('set', { + type: 'Set', + elements: [ + { + type: 'Array', + elements: [ + { type: 'number', value: '1' }, + { type: 'number', value: '2' } + ] + }, + { type: 'number', value: '3' }, + { type: 'number', value: '4' } + ] + }) + }) + + it('WeakMap', function () { + expect(state).to.have.property('wmap') + expect(state.wmap).to.have.keys('type', 'entries') + expect(state.wmap.entries).to.be.an('array') + state.wmap.entries = state.wmap.entries.sort((a, b) => a[1].value - b[1].value) + expect(state).to.have.deep.property('wmap', { + type: 'WeakMap', + entries: [[ + { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + { type: 'number', value: '2' } + ], [ + { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + { type: 'number', value: '4' } + ]] + }) + }) + + it('WeakSet', function () { + expect(state).to.have.property('wset') + expect(state.wset).to.have.keys('type', 'elements') + expect(state.wset.elements).to.be.an('array') + state.wset.elements = state.wset.elements.sort((a, b) => a.fields.a.value - b.fields.a.value) + expect(state).to.have.deep.property('wset', { + type: 'WeakSet', + elements: [ + { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + ] + }) + }) + + it('Generator', function () { + expect(state).to.have.deep.property('gen', { + type: 'generator', + fields: { foo: { type: 'number', value: '42' } } + }) + }) + + it('Error', function () { + expect(state).to.have.property('err') + expect(state.err).to.have.keys('type', 'fields') + expect(state.err).to.have.property('type', 'CustomError') + expect(state.err.fields).to.be.an('object') + expect(state.err.fields).to.have.keys('stack', 'message', 'foo') + expect(state.err.fields).to.deep.include({ + message: { type: 'string', value: 'boom!' }, + foo: { type: 'number', value: '42' } + }) + expect(state.err.fields.stack).to.have.keys('type', 'value', 'truncated', 'size') + expect(state.err.fields.stack.value).to.be.a('string') + expect(state.err.fields.stack.value).to.match(/^Error: boom!/) + expect(state.err.fields.stack.size).to.be.a('number') + expect(state.err.fields.stack.size).to.above(255) + expect(state.err.fields.stack).to.deep.include({ + type: 'string', + truncated: true + }) + }) + + it('Function', function () { + expect(state).to.have.deep.property('fn', { + type: 'Function', + fields: { + foo: { + type: 'Object', + fields: { bar: { type: 'number', value: '42' } } + }, + length: { type: 'number', value: '2' }, + name: { type: 'string', value: 'fnWithProperties' } + } + }) + }) + + it('Bound function', function () { + expect(state).to.have.deep.property('bfn', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'bound fnWithProperties' } + } + }) + }) + + it('Arrow function', function () { + expect(state).to.have.deep.property('afn', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'afn' } + } + }) + }) + + it('Class', function () { + expect(state).to.have.deep.property('cls', { type: 'class MyClass' }) + }) + + it('Anonymous class', function () { + expect(state).to.have.deep.property('acls', { type: 'class' }) + }) + + it('Proxy for object literal', function () { + expect(state).to.have.deep.property('prox', { + type: NODE_20_PLUS ? 'Proxy(Object)' : 'Proxy', + fields: { + target: { type: 'boolean', value: 'true' } + } + }) + }) + + it('Proxy for custom class', function () { + expect(state).to.have.deep.property('custProx', { + type: NODE_20_PLUS ? 'Proxy(MyClass)' : 'Proxy', + fields: { + foo: { type: 'number', value: '42' } + } + }) + }) + + it('Promise: Pending', function () { + expect(state).to.have.deep.property('pPen', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'pending' }, + '[[PromiseResult]]': { type: 'undefined' } + } + }) + }) + + it('Promise: Resolved', function () { + expect(state).to.have.deep.property('pRes', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'fulfilled' }, + '[[PromiseResult]]': { type: 'string', value: 'resolved value' } + } + }) + }) + + it('Promise: Rejected', function () { + expect(state).to.have.deep.property('pRej', { + type: 'Promise', + fields: { + '[[PromiseState]]': { type: 'string', value: 'rejected' }, + '[[PromiseResult]]': { type: 'string', value: 'rejected value' } + } + }) + }) + + it('TypedArray', function () { + expect(state).to.have.deep.property('tarr', { + type: 'Int8Array', + elements: [ + { type: 'number', value: '72' }, + { type: 'number', value: '65' }, + { type: 'number', value: '76' } + ] + }) + }) + + it('ArrayBuffer', function () { + expect(state).to.have.deep.property('ab', { + type: 'ArrayBuffer', + value: 'HAL' + }) + }) + + it('SharedArrayBuffer', function () { + expect(state).to.have.deep.property('sab', { + type: 'SharedArrayBuffer', + value: 'hello\x01\x02\x03world' + }) + }) + + it('circular reference in object', function () { + expect(state).to.have.property('circular') + expect(state.circular).to.have.property('type', 'Object') + expect(state.circular).to.have.property('fields') + // For the circular field, just check that at least one of the expected properties are present + expect(state.circular.fields).to.deep.include({ + regex: { type: 'RegExp', value: '/foo/' } + }) + }) + + it('non-enumerable property', function () { + expect(state).to.have.deep.property('hidden', { type: 'string', value: 'secret' }) + }) + }) + + it('should return expected object for nested objects with maxReferenceDepth: 1', async function () { + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + const state = (await getLocalStateForCallFrame(params.callFrames[0], { maxReferenceDepth: 1 }))() + + expect(Object.keys(state).length).to.equal(5) + + // from block scope + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.keys(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', notCapturedReason: 'depth' + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', notCapturedReason: 'depth' + }) + + // from local scope + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', notCapturedReason: 'depth' }, + wmo2: { type: 'Object', notCapturedReason: 'depth' }, + wso1: { type: 'Object', notCapturedReason: 'depth' }, + wso2: { type: 'Object', notCapturedReason: 'depth' }, + wso3: { type: 'Object', notCapturedReason: 'depth' } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + await setBreakpointOnLine(18) + getNestedObj() + }) + + it('should return expected object for nested objects with maxReferenceDepth: 5', async function () { + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + const state = (await getLocalStateForCallFrame(params.callFrames[0], { maxReferenceDepth: 5 }))() + + expect(Object.entries(state).length).to.equal(5) + + // from block scope + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.entries(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { type: 'Object', notCapturedReason: 'depth' } + } + } + } + } + } + } + } + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ type: 'Array', notCapturedReason: 'depth' }] + }] + }] + }] + }) + + // from local scope + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wmo2: { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + wso1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wso2: { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + wso3: { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + await setBreakpointOnLine(18) + getNestedObj() + }) + + it('should return expected object for nested objects if maxReferenceDepth is missing', async function () { + session.once('Debugger.paused', async ({ params }) => { + expect(params.hitBreakpoints.length).to.eq(1) + + const state = (await getLocalStateForCallFrame(params.callFrames[0]))() + + expect(Object.entries(state).length).to.equal(5) + + // from block scope + expect(state).to.have.property('myNestedObj') + expect(state.myNestedObj).to.have.property('type', 'Object') + expect(state.myNestedObj).to.have.property('fields') + expect(Object.entries(state.myNestedObj).length).to.equal(2) + + expect(state.myNestedObj.fields).to.have.deep.property('deepObj', { + type: 'Object', + fields: { + foo: { + type: 'Object', + fields: { + foo: { + type: 'Object', + notCapturedReason: 'depth' + } + } + } + } + }) + + expect(state.myNestedObj.fields).to.have.deep.property('deepArr', { + type: 'Array', + elements: [{ + type: 'Array', + elements: [{ + type: 'Array', + notCapturedReason: 'depth' + }] + }] + }) + + // from local scope + expect(state).to.have.deep.property('a1', { type: 'number', value: '1' }) + expect(state).to.have.deep.property('a2', { type: 'number', value: '2' }) + + // from closure scope + expect(state).to.have.deep.property('ref', { + type: 'Object', + fields: { + wmo1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wmo2: { type: 'Object', fields: { b: { type: 'number', value: '3' } } }, + wso1: { type: 'Object', fields: { a: { type: 'number', value: '1' } } }, + wso2: { type: 'Object', fields: { a: { type: 'number', value: '2' } } }, + wso3: { type: 'Object', fields: { a: { type: 'number', value: '3' } } } + } + }) + expect(state).to.have.deep.property('get', { + type: 'Function', + fields: { + length: { type: 'number', value: '0' }, + name: { type: 'string', value: 'get' } + } + }) + }) + + await setBreakpointOnLine(18) + getNestedObj() + }) +}) + +async function setBreakpointOnLine (line) { + await session.post('Debugger.setBreakpoint', { + location: { + scriptId: await scriptId, + lineNumber: line - 1 // Beware! lineNumber is zero-indexed + } + }) +} From 08525d4c3c63088a21505d86a8a97e1bc537d244 Mon Sep 17 00:00:00 2001 From: William Conti <58711692+wconti27@users.noreply.github.com> Date: Fri, 4 Oct 2024 10:57:43 -0400 Subject: [PATCH 18/27] feat(tracing): implement protobufjs DSM schema support (#4701) * add protobufjs schemas support for DSM --- .github/workflows/plugins.yml | 9 + docs/API.md | 2 + docs/add-redirects.sh | 1 + docs/test.ts | 1 + index.d.ts | 7 + .../src/helpers/hooks.js | 1 + .../src/protobufjs.js | 127 +++++++ .../datadog-plugin-protobufjs/src/index.js | 14 + .../src/schema_iterator.js | 183 +++++++++ .../datadog-plugin-protobufjs/test/helpers.js | 104 ++++++ .../test/index.spec.js | 352 ++++++++++++++++++ .../test/schemas/all_types.proto | 49 +++ .../test/schemas/expected_schemas.json | 195 ++++++++++ .../test/schemas/message.proto | 17 + .../test/schemas/other_message.proto | 6 + .../test/schemas/other_message_proto.json | 17 + .../src/datastreams/schemas/schema_builder.js | 42 ++- packages/dd-trace/src/plugins/index.js | 1 + packages/dd-trace/src/plugins/schema.js | 35 ++ .../schemas/schema_builder.spec.js | 2 +- 20 files changed, 1147 insertions(+), 18 deletions(-) create mode 100644 packages/datadog-instrumentations/src/protobufjs.js create mode 100644 packages/datadog-plugin-protobufjs/src/index.js create mode 100644 packages/datadog-plugin-protobufjs/src/schema_iterator.js create mode 100644 packages/datadog-plugin-protobufjs/test/helpers.js create mode 100644 packages/datadog-plugin-protobufjs/test/index.spec.js create mode 100644 packages/datadog-plugin-protobufjs/test/schemas/all_types.proto create mode 100644 packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json create mode 100644 packages/datadog-plugin-protobufjs/test/schemas/message.proto create mode 100644 packages/datadog-plugin-protobufjs/test/schemas/other_message.proto create mode 100644 packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json create mode 100644 packages/dd-trace/src/plugins/schema.js diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 405bc562f0e..60872f79740 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -803,6 +803,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test + protobufjs: + runs-on: ubuntu-latest + env: + PLUGINS: protobufjs + DD_DATA_STREAMS_ENABLED: true + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream + q: runs-on: ubuntu-latest env: diff --git a/docs/API.md b/docs/API.md index 68cdc3747cb..713c17032cf 100644 --- a/docs/API.md +++ b/docs/API.md @@ -87,6 +87,7 @@ tracer.use('pg', {
+
@@ -142,6 +143,7 @@ tracer.use('pg', { * [pg](./interfaces/export_.plugins.pg.html) * [promise](./interfaces/export_.plugins.promise.html) * [promise-js](./interfaces/export_.plugins.promise_js.html) +* [protobufjs](./interfaces/export_.plugins.protobufjs.html) * [q](./interfaces/export_.plugins.q.html) * [redis](./interfaces/export_.plugins.redis.html) * [restify](./interfaces/export_.plugins.restify.html) diff --git a/docs/add-redirects.sh b/docs/add-redirects.sh index fd0590a934a..732c8a83607 100755 --- a/docs/add-redirects.sh +++ b/docs/add-redirects.sh @@ -55,6 +55,7 @@ declare -a plugins=( "pg" "promise" "promise_js" + "protobufjs" "q" "redis" "restify" diff --git a/docs/test.ts b/docs/test.ts index e37177e0898..c45d0f3d515 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -362,6 +362,7 @@ tracer.use('playwright'); tracer.use('pg'); tracer.use('pg', { service: params => `${params.host}-${params.database}` }); tracer.use('pino'); +tracer.use('protobufjs'); tracer.use('redis'); tracer.use('redis', redisOptions); tracer.use('restify'); diff --git a/index.d.ts b/index.d.ts index 02c84fb47d3..115b123e569 100644 --- a/index.d.ts +++ b/index.d.ts @@ -190,6 +190,7 @@ interface Plugins { "playwright": tracer.plugins.playwright; "pg": tracer.plugins.pg; "pino": tracer.plugins.pino; + "protobufjs": tracer.plugins.protobufjs; "redis": tracer.plugins.redis; "restify": tracer.plugins.restify; "rhea": tracer.plugins.rhea; @@ -1731,6 +1732,12 @@ declare namespace tracer { * on the tracer. */ interface pino extends Integration {} + + /** + * This plugin automatically patches the [protobufjs](https://protobufjs.github.io/protobuf.js/) + * to collect protobuf message schemas when Datastreams Monitoring is enabled. + */ + interface protobufjs extends Integration {} /** * This plugin automatically instruments the diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index 284e4ed5950..eb2cbcb794c 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -100,6 +100,7 @@ module.exports = { playwright: () => require('../playwright'), 'promise-js': () => require('../promise-js'), promise: () => require('../promise'), + protobufjs: () => require('../protobufjs'), q: () => require('../q'), qs: () => require('../qs'), redis: () => require('../redis'), diff --git a/packages/datadog-instrumentations/src/protobufjs.js b/packages/datadog-instrumentations/src/protobufjs.js new file mode 100644 index 00000000000..79cbb4ee3a1 --- /dev/null +++ b/packages/datadog-instrumentations/src/protobufjs.js @@ -0,0 +1,127 @@ +const shimmer = require('../../datadog-shimmer') +const { addHook } = require('./helpers/instrument') + +const dc = require('dc-polyfill') +const serializeChannel = dc.channel('apm:protobufjs:serialize-start') +const deserializeChannel = dc.channel('apm:protobufjs:deserialize-end') + +function wrapSerialization (messageClass) { + if (messageClass?.encode) { + shimmer.wrap(messageClass, 'encode', original => function () { + if (!serializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + serializeChannel.publish({ messageClass: this }) + return original.apply(this, arguments) + }) + } +} + +function wrapDeserialization (messageClass) { + if (messageClass?.decode) { + shimmer.wrap(messageClass, 'decode', original => function () { + if (!deserializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + const result = original.apply(this, arguments) + deserializeChannel.publish({ messageClass: result }) + return result + }) + } +} + +function wrapSetup (messageClass) { + if (messageClass?.setup) { + shimmer.wrap(messageClass, 'setup', original => function () { + const result = original.apply(this, arguments) + + wrapSerialization(messageClass) + wrapDeserialization(messageClass) + + return result + }) + } +} + +function wrapProtobufClasses (root) { + if (!root) { + return + } + + if (root.decode) { + wrapSetup(root) + } + + if (root.nestedArray) { + for (const subRoot of root.nestedArray) { + wrapProtobufClasses(subRoot) + } + } +} + +function wrapReflection (protobuf) { + const reflectionMethods = [ + { + target: protobuf.Root, + name: 'fromJSON' + }, + { + target: protobuf.Type.prototype, + name: 'fromObject' + } + ] + + reflectionMethods.forEach(method => { + shimmer.wrap(method.target, method.name, original => function () { + const result = original.apply(this, arguments) + if (result.nested) { + for (const type in result.nested) { + wrapSetup(result.nested[type]) + } + } + if (result.$type) { + wrapSetup(result.$type) + } + return result + }) + }) +} + +function isPromise (obj) { + return !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function' +} + +addHook({ + name: 'protobufjs', + versions: ['>=6.8.0'] +}, protobuf => { + shimmer.wrap(protobuf.Root.prototype, 'load', original => function () { + const result = original.apply(this, arguments) + if (isPromise(result)) { + return result.then(root => { + wrapProtobufClasses(root) + return root + }) + } else { + // If result is not a promise, directly wrap the protobuf classes + wrapProtobufClasses(result) + return result + } + }) + + shimmer.wrap(protobuf.Root.prototype, 'loadSync', original => function () { + const root = original.apply(this, arguments) + wrapProtobufClasses(root) + return root + }) + + shimmer.wrap(protobuf, 'Type', Original => function () { + const typeInstance = new Original(...arguments) + wrapSetup(typeInstance) + return typeInstance + }) + + wrapReflection(protobuf) + + return protobuf +}) diff --git a/packages/datadog-plugin-protobufjs/src/index.js b/packages/datadog-plugin-protobufjs/src/index.js new file mode 100644 index 00000000000..800c3d9e3cb --- /dev/null +++ b/packages/datadog-plugin-protobufjs/src/index.js @@ -0,0 +1,14 @@ +const SchemaPlugin = require('../../dd-trace/src/plugins/schema') +const SchemaExtractor = require('./schema_iterator') + +class ProtobufjsPlugin extends SchemaPlugin { + static get id () { + return 'protobufjs' + } + + static get schemaExtractor () { + return SchemaExtractor + } +} + +module.exports = ProtobufjsPlugin diff --git a/packages/datadog-plugin-protobufjs/src/schema_iterator.js b/packages/datadog-plugin-protobufjs/src/schema_iterator.js new file mode 100644 index 00000000000..a06f7eb313a --- /dev/null +++ b/packages/datadog-plugin-protobufjs/src/schema_iterator.js @@ -0,0 +1,183 @@ +const PROTOBUF = 'protobuf' +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const { + SchemaBuilder +} = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +class SchemaExtractor { + constructor (schema) { + this.schema = schema + } + + static getTypeAndFormat (type) { + const typeFormatMapping = { + int32: ['integer', 'int32'], + int64: ['integer', 'int64'], + uint32: ['integer', 'uint32'], + uint64: ['integer', 'uint64'], + sint32: ['integer', 'sint32'], + sint64: ['integer', 'sint64'], + fixed32: ['integer', 'fixed32'], + fixed64: ['integer', 'fixed64'], + sfixed32: ['integer', 'sfixed32'], + sfixed64: ['integer', 'sfixed64'], + float: ['number', 'float'], + double: ['number', 'double'], + bool: ['boolean', null], + string: ['string', null], + bytes: ['string', 'byte'], + Enum: ['enum', null], + Type: ['type', null], + map: ['map', null], + repeated: ['array', null] + } + + return typeFormatMapping[type] || ['string', null] + } + + static extractProperty (field, schemaName, fieldName, builder, depth) { + let array = false + let description + let ref + let enumValues + + const resolvedType = field.resolvedType ? field.resolvedType.constructor.name : field.type + + const isRepeatedField = field.rule === 'repeated' + + let typeFormat = this.getTypeAndFormat(isRepeatedField ? 'repeated' : resolvedType) + let type = typeFormat[0] + let format = typeFormat[1] + + if (type === 'array') { + array = true + typeFormat = this.getTypeAndFormat(resolvedType) + type = typeFormat[0] + format = typeFormat[1] + } + + if (type === 'type') { + format = null + ref = `#/components/schemas/${removeLeadingPeriod(field.resolvedType.fullName)}` + // keep a reference to the original builder iterator since when we recurse this reference will get reset to + // deeper schemas + const originalSchemaExtractor = builder.iterator + if (!this.extractSchema(field.resolvedType, builder, depth, this)) { + return false + } + type = 'object' + builder.iterator = originalSchemaExtractor + } else if (type === 'enum') { + enumValues = [] + let i = 0 + while (field.resolvedType.valuesById[i]) { + enumValues.push(field.resolvedType.valuesById[i]) + i += 1 + } + } + return builder.addProperty(schemaName, fieldName, array, type, description, ref, format, enumValues) + } + + static extractSchema (schema, builder, depth, extractor) { + depth += 1 + const schemaName = removeLeadingPeriod(schema.resolvedType ? schema.resolvedType.fullName : schema.fullName) + if (extractor) { + // if we already have a defined extractor, this is a nested schema. create a new extractor for the nested + // schema, ensure it is added to our schema builder's cache, and replace the builders iterator with our + // nested schema iterator / extractor. Once complete, add the new schema to our builder's schemas. + const nestedSchemaExtractor = new SchemaExtractor(schema) + builder.iterator = nestedSchemaExtractor + const nestedSchema = SchemaBuilder.getSchema(schemaName, nestedSchemaExtractor, builder) + for (const nestedSubSchemaName in nestedSchema.components.schemas) { + if (nestedSchema.components.schemas.hasOwnProperty(nestedSubSchemaName)) { + builder.schema.components.schemas[nestedSubSchemaName] = nestedSchema.components.schemas[nestedSubSchemaName] + } + } + return true + } else { + if (!builder.shouldExtractSchema(schemaName, depth)) { + return false + } + try { + for (const field of schema.fieldsArray) { + if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { + return false + } + } + } catch (error) { + return false + } + return true + } + } + + static extractSchemas (descriptor, dataStreamsProcessor) { + const schemaName = removeLeadingPeriod( + descriptor.resolvedType ? descriptor.resolvedType.fullName : descriptor.fullName + ) + return dataStreamsProcessor.getSchema(schemaName, new SchemaExtractor(descriptor)) + } + + iterateOverSchema (builder) { + this.constructor.extractSchema(this.schema, builder, 0) + } + + static attachSchemaOnSpan (args, span, operation, tracer) { + const { messageClass } = args + const descriptor = messageClass.$type ?? messageClass + + if (!descriptor || !span) { + return + } + + if (span.context()._tags[SCHEMA_TYPE] && operation === 'serialization') { + // we have already added a schema to this span, this call is an encode of nested schema types + return + } + + span.setTag(SCHEMA_TYPE, PROTOBUF) + span.setTag(SCHEMA_NAME, removeLeadingPeriod(descriptor.fullName)) + span.setTag(SCHEMA_OPERATION, operation) + + if (!tracer._dataStreamsProcessor.canSampleSchema(operation)) { + return + } + + // if the span is unsampled, do not sample the schema + if (!tracer._prioritySampler.isSampled(span)) { + return + } + + const weight = tracer._dataStreamsProcessor.trySampleSchema(operation) + if (weight === 0) { + return + } + + const schemaData = SchemaBuilder.getSchemaDefinition( + this.extractSchemas(descriptor, tracer._dataStreamsProcessor) + ) + + span.setTag(SCHEMA_DEFINITION, schemaData.definition) + span.setTag(SCHEMA_WEIGHT, weight) + span.setTag(SCHEMA_ID, schemaData.id) + } +} + +function removeLeadingPeriod (str) { + // Check if the first character is a period + if (str.charAt(0) === '.') { + // Remove the first character + return str.slice(1) + } + // Return the original string if the first character is not a period + return str +} + +module.exports = SchemaExtractor diff --git a/packages/datadog-plugin-protobufjs/test/helpers.js b/packages/datadog-plugin-protobufjs/test/helpers.js new file mode 100644 index 00000000000..d91be2e496b --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/helpers.js @@ -0,0 +1,104 @@ +async function loadMessage (protobuf, messageTypeName) { + if (messageTypeName === 'OtherMessage') { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + return { + OtherMessage: { + type: OtherMessage, + instance: message + } + } + } else if (messageTypeName === 'MyMessage') { + const messageProto = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/message.proto') + const otherMessageProto = await protobuf.load( + 'packages/datadog-plugin-protobufjs/test/schemas/other_message.proto' + ) + const Status = messageProto.lookupEnum('Status') + const MyMessage = messageProto.lookupType('MyMessage') + const OtherMessage = otherMessageProto.lookupType('OtherMessage') + const message = MyMessage.create({ + id: '123', + value: 'example_value', + status: Status.values.ACTIVE, + otherMessage: [ + OtherMessage.create({ name: ['Alice'], age: 30 }), + OtherMessage.create({ name: ['Bob'], age: 25 }) + ] + }) + return { + OtherMessage: { + type: OtherMessage, + instance: null + }, + MyMessage: { + type: MyMessage, + instance: message + } + } + } else if (messageTypeName === 'MainMessage') { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/all_types.proto') + + const Status = root.lookupEnum('example.Status') + const Scalars = root.lookupType('example.Scalars') + const NestedMessage = root.lookupType('example.NestedMessage') + const ComplexMessage = root.lookupType('example.ComplexMessage') + const MainMessage = root.lookupType('example.MainMessage') + + // Create instances of the messages + const scalarsInstance = Scalars.create({ + int32Field: 42, + int64Field: 123456789012345, + uint32Field: 123, + uint64Field: 123456789012345, + sint32Field: -42, + sint64Field: -123456789012345, + fixed32Field: 42, + fixed64Field: 123456789012345, + sfixed32Field: -42, + sfixed64Field: -123456789012345, + floatField: 3.14, + doubleField: 2.718281828459, + boolField: true, + stringField: 'Hello, world!', + bytesField: Buffer.from('bytes data') + }) + + const nestedMessageInstance = NestedMessage.create({ + id: 'nested_id_123', + scalars: scalarsInstance + }) + + const complexMessageInstance = ComplexMessage.create({ + repeatedField: ['item1', 'item2', 'item3'], + mapField: { + key1: scalarsInstance, + key2: Scalars.create({ + int32Field: 24, + stringField: 'Another string' + }) + } + }) + + const mainMessageInstance = MainMessage.create({ + status: Status.values.ACTIVE, + scalars: scalarsInstance, + nested: nestedMessageInstance, + complex: complexMessageInstance + }) + + return { + MainMessage: { + type: MainMessage, + instance: mainMessageInstance + } + } + } +} + +module.exports = { + loadMessage +} diff --git a/packages/datadog-plugin-protobufjs/test/index.spec.js b/packages/datadog-plugin-protobufjs/test/index.spec.js new file mode 100644 index 00000000000..30e95687bac --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/index.spec.js @@ -0,0 +1,352 @@ +'use strict' + +const fs = require('fs') +const { expect } = require('chai') +const agent = require('../../dd-trace/test/plugins/agent') +const path = require('path') +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const sinon = require('sinon') +const { loadMessage } = require('./helpers') +const { SchemaBuilder } = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +const schemas = JSON.parse(fs.readFileSync(path.join(__dirname, 'schemas/expected_schemas.json'), 'utf8')) +const MESSAGE_SCHEMA_DEF = schemas.MESSAGE_SCHEMA_DEF +const OTHER_MESSAGE_SCHEMA_DEF = schemas.OTHER_MESSAGE_SCHEMA_DEF +const ALL_TYPES_MESSAGE_SCHEMA_DEF = schemas.ALL_TYPES_MESSAGE_SCHEMA_DEF + +const MESSAGE_SCHEMA_ID = '666607144722735562' +const OTHER_MESSAGE_SCHEMA_ID = '2691489402935632768' +const ALL_TYPES_MESSAGE_SCHEMA_ID = '15890948796193489151' + +function compareJson (expected, span) { + const actual = JSON.parse(span.context()._tags[SCHEMA_DEFINITION]) + return JSON.stringify(actual) === JSON.stringify(expected) +} + +describe('Plugin', () => { + describe('protobufjs', function () { + let tracer + let protobuf + let dateNowStub + let mockTime = 0 + + withVersions('protobufjs', ['protobufjs'], (version) => { + before(() => { + tracer = require('../../dd-trace').init() + // reset sampled schemas + if (tracer._dataStreamsProcessor?._schemaSamplers) { + tracer._dataStreamsProcessor._schemaSamplers = [] + } + }) + + describe('without configuration', () => { + before(() => { + dateNowStub = sinon.stub(Date, 'now').callsFake(() => { + const returnValue = mockTime + mockTime += 50000 // Increment by 50000 ms to ensure each DSM schema is sampled + return returnValue + }) + const cache = SchemaBuilder.getCache() + cache.clear() + return agent.load('protobufjs').then(() => { + protobuf = require(`../../../versions/protobufjs@${version}`).get() + }) + }) + + after(() => { + dateNowStub.restore() + return agent.close({ ritmReset: false }) + }) + + it('should serialize basic schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'OtherMessage') + + tracer.trace('other_message.serialize', span => { + loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + expect(span._name).to.equal('other_message.serialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using a callback instead of promise', async () => { + const loadedMessages = loadMessage(protobuf, 'OtherMessage', () => { + tracer.trace('other_message.serialize', span => { + loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + expect(span._name).to.equal('other_message.serialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + }) + + it('should serialize complex schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should serialize schema with all types correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MainMessage') + + tracer.trace('all_types.serialize', span => { + loadedMessages.MainMessage.type.encode(loadedMessages.MainMessage.instance).finish() + + expect(span._name).to.equal('all_types.serialize') + + expect(compareJson(ALL_TYPES_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.MainMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ALL_TYPES_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize basic schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'OtherMessage') + + const bytes = loadedMessages.OtherMessage.type.encode(loadedMessages.OtherMessage.instance).finish() + + tracer.trace('other_message.deserialize', span => { + loadedMessages.OtherMessage.type.decode(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize complex schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + const bytes = loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + tracer.trace('my_message.deserialize', span => { + loadedMessages.MyMessage.type.decode(bytes) + + expect(span._name).to.equal('my_message.deserialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize all types schema correctly', async () => { + const loadedMessages = await loadMessage(protobuf, 'MainMessage') + + const bytes = loadedMessages.MainMessage.type.encode(loadedMessages.MainMessage.instance).finish() + + tracer.trace('all_types.deserialize', span => { + loadedMessages.MainMessage.type.decode(bytes) + + expect(span._name).to.equal('all_types.deserialize') + + expect(compareJson(ALL_TYPES_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.MainMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ALL_TYPES_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should wrap encode and decode for fromObject', async () => { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const messageObject = { + name: ['Alice'], + age: 30 + } + const message = OtherMessage.fromObject(messageObject) + + const bytes = OtherMessage.encode(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decode(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should wrap decodeDelimited', async () => { + const root = await protobuf.load('packages/datadog-plugin-protobufjs/test/schemas/other_message.proto') + const OtherMessage = root.lookupType('OtherMessage') + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using direct type creation', () => { + const OtherMessage = new protobuf.Type('OtherMessage') + .add(new protobuf.Field('name', 1, 'string', 'repeated')) + .add(new protobuf.Field('age', 2, 'int32')) + + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should load using JSON descriptors', () => { + const jsonDescriptor = require('./schemas/other_message_proto.json') + const root = protobuf.Root.fromJSON(jsonDescriptor) + const OtherMessage = root.lookupType('OtherMessage') + + const message = OtherMessage.create({ + name: ['Alice'], + age: 30 + }) + + const bytes = OtherMessage.encodeDelimited(message).finish() + + tracer.trace('other_message.deserialize', span => { + OtherMessage.decodeDelimited(bytes) + + expect(span._name).to.equal('other_message.deserialize') + + expect(compareJson(OTHER_MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'OtherMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, OTHER_MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + describe('during schema sampling', function () { + let cacheSetSpy + let cacheGetSpy + + beforeEach(() => { + const cache = SchemaBuilder.getCache() + cache.clear() + cacheSetSpy = sinon.spy(cache, 'set') + cacheGetSpy = sinon.spy(cache, 'get') + }) + + afterEach(() => { + cacheSetSpy.restore() + cacheGetSpy.restore() + }) + + it('should use the schema cache and not re-extract an already sampled schema', async () => { + const loadedMessages = await loadMessage(protobuf, 'MyMessage') + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + + // we sampled 1 schema with 1 subschema, so the constructor should've only been called twice + expect(cacheSetSpy.callCount).to.equal(2) + expect(cacheGetSpy.callCount).to.equal(2) + }) + + tracer.trace('message_pb2.serialize', span => { + loadedMessages.MyMessage.type.encode(loadedMessages.MyMessage.instance).finish() + + expect(span._name).to.equal('message_pb2.serialize') + + expect(compareJson(MESSAGE_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'protobuf') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'MyMessage') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, MESSAGE_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + + // ensure schema was sampled and returned via the cache, so no extra cache set + // calls were needed, only gets + expect(cacheSetSpy.callCount).to.equal(2) + expect(cacheGetSpy.callCount).to.equal(3) + }) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto b/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto new file mode 100644 index 00000000000..6cfc3b3ee3d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/all_types.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +package example; + +// Enum definition +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; +} + +// Message with various number types and other scalar types +message Scalars { + int32 int32Field = 1; + int64 int64Field = 2; + uint32 uint32Field = 3; + uint64 uint64Field = 4; + sint32 sint32Field = 5; + sint64 sint64Field = 6; + fixed32 fixed32Field = 7; + fixed64 fixed64Field = 8; + sfixed32 sfixed32Field = 9; + sfixed64 sfixed64Field = 10; + float floatField = 11; + double doubleField = 12; + bool boolField = 13; + string stringField = 14; + bytes bytesField = 15; +} + +// Nested message definition +message NestedMessage { + string id = 1; + Scalars scalars = 2; +} + +// Message demonstrating the use of repeated fields and maps +message ComplexMessage { + repeated string repeatedField = 1; + map mapField = 2; +} + +// Main message that uses all the above elements +message MainMessage { + Status status = 1; + Scalars scalars = 2; + NestedMessage nested = 3; + ComplexMessage complex = 4; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json b/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json new file mode 100644 index 00000000000..1825013519d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/expected_schemas.json @@ -0,0 +1,195 @@ +{ + "MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "MyMessage":{ + "type":"object", + "properties":{ + "id":{ + "type":"string" + }, + "value":{ + "type":"string" + }, + "otherMessage":{ + "type":"array", + "items":{ + "type":"object", + "$ref":"#/components/schemas/OtherMessage" + } + }, + "status":{ + "type":"enum", + "enum":[ + "UNKNOWN", + "ACTIVE", + "INACTIVE", + "DELETED" + ] + } + } + }, + "OtherMessage":{ + "type":"object", + "properties":{ + "name":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "age":{ + "type":"integer", + "format":"int32" + } + } + } + } + } + }, + "OTHER_MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "OtherMessage":{ + "type":"object", + "properties":{ + "name":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "age":{ + "type":"integer", + "format":"int32" + } + } + } + } + } + }, + "ALL_TYPES_MESSAGE_SCHEMA_DEF":{ + "openapi":"3.0.0", + "components":{ + "schemas":{ + "example.MainMessage":{ + "type":"object", + "properties":{ + "status":{ + "type":"enum", + "enum":[ + "UNKNOWN", + "ACTIVE", + "INACTIVE" + ] + }, + "scalars":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + }, + "nested":{ + "type":"object", + "$ref":"#/components/schemas/example.NestedMessage" + }, + "complex":{ + "type":"object", + "$ref":"#/components/schemas/example.ComplexMessage" + } + } + }, + "example.Scalars":{ + "type":"object", + "properties":{ + "int32Field":{ + "type":"integer", + "format":"int32" + }, + "int64Field":{ + "type":"integer", + "format":"int64" + }, + "uint32Field":{ + "type":"integer", + "format":"uint32" + }, + "uint64Field":{ + "type":"integer", + "format":"uint64" + }, + "sint32Field":{ + "type":"integer", + "format":"sint32" + }, + "sint64Field":{ + "type":"integer", + "format":"sint64" + }, + "fixed32Field":{ + "type":"integer", + "format":"fixed32" + }, + "fixed64Field":{ + "type":"integer", + "format":"fixed64" + }, + "sfixed32Field":{ + "type":"integer", + "format":"sfixed32" + }, + "sfixed64Field":{ + "type":"integer", + "format":"sfixed64" + }, + "floatField":{ + "type":"number", + "format":"float" + }, + "doubleField":{ + "type":"number", + "format":"double" + }, + "boolField":{ + "type":"boolean" + }, + "stringField":{ + "type":"string" + }, + "bytesField":{ + "type":"string", + "format":"byte" + } + } + }, + "example.NestedMessage":{ + "type":"object", + "properties":{ + "id":{ + "type":"string" + }, + "scalars":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + } + } + }, + "example.ComplexMessage":{ + "type":"object", + "properties":{ + "repeatedField":{ + "type":"array", + "items":{ + "type":"string" + } + }, + "mapField":{ + "type":"object", + "$ref":"#/components/schemas/example.Scalars" + } + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/message.proto b/packages/datadog-plugin-protobufjs/test/schemas/message.proto new file mode 100644 index 00000000000..6fd1c65fe06 --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/message.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +import "other_message.proto"; + +enum Status { + UNKNOWN = 0; + ACTIVE = 1; + INACTIVE = 2; + DELETED = 3; +} + +message MyMessage { + string id = 1; + string value = 2; + repeated OtherMessage otherMessage = 3; + Status status = 4; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto b/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto new file mode 100644 index 00000000000..dbd6f368d7d --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/other_message.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message OtherMessage { + repeated string name = 1; + int32 age = 2; +} \ No newline at end of file diff --git a/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json b/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json new file mode 100644 index 00000000000..5a682ec89ca --- /dev/null +++ b/packages/datadog-plugin-protobufjs/test/schemas/other_message_proto.json @@ -0,0 +1,17 @@ +{ + "nested": { + "OtherMessage": { + "fields": { + "name": { + "rule": "repeated", + "type": "string", + "id": 1 + }, + "age": { + "type": "int32", + "id": 2 + } + } + } + } + } \ No newline at end of file diff --git a/packages/dd-trace/src/datastreams/schemas/schema_builder.js b/packages/dd-trace/src/datastreams/schemas/schema_builder.js index a65863d4d87..092f5b45101 100644 --- a/packages/dd-trace/src/datastreams/schemas/schema_builder.js +++ b/packages/dd-trace/src/datastreams/schemas/schema_builder.js @@ -4,13 +4,36 @@ const { Schema } = require('./schema') const maxDepth = 10 const maxProperties = 1000 -const CACHE = new LRUCache({ max: 32 }) +const CACHE = new LRUCache({ max: 256 }) class SchemaBuilder { constructor (iterator) { this.schema = new OpenApiSchema() this.iterator = iterator - this.proerties = 0 + this.properties = 0 + } + + static getCache () { + return CACHE + } + + static getSchemaDefinition (schema) { + const noNones = convertToJsonCompatible(schema) + const definition = jsonStringify(noNones) + const id = fnv64(Buffer.from(definition, 'utf-8')).toString() + return new Schema(definition, id) + } + + static getSchema (schemaName, iterator, builder) { + if (!CACHE.has(schemaName)) { + CACHE.set(schemaName, (builder ?? new SchemaBuilder(iterator)).build()) + } + return CACHE.get(schemaName) + } + + build () { + this.iterator.iterateOverSchema(this) + return this.schema } addProperty (schemaName, fieldName, isArray, type, description, ref, format, enumValues) { @@ -26,14 +49,6 @@ class SchemaBuilder { return true } - build () { - this.iterator.iterateOverSchema(this) - const noNones = convertToJsonCompatible(this.schema) - const definition = jsonStringify(noNones) - const id = fnv64(Buffer.from(definition, 'utf-8')).toString() - return new Schema(definition, id) - } - shouldExtractSchema (schemaName, depth) { if (depth > maxDepth) { return false @@ -44,13 +59,6 @@ class SchemaBuilder { this.schema.components.schemas[schemaName] = new OpenApiSchema.SCHEMA() return true } - - static getSchema (schemaName, iterator) { - if (!CACHE.has(schemaName)) { - CACHE.set(schemaName, new SchemaBuilder(iterator).build()) - } - return CACHE.get(schemaName) - } } class OpenApiSchema { diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 06325724b71..2e949fff7e2 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -77,6 +77,7 @@ module.exports = { get pino () { return require('../../../datadog-plugin-pino/src') }, get 'pino-pretty' () { return require('../../../datadog-plugin-pino/src') }, get playwright () { return require('../../../datadog-plugin-playwright/src') }, + get protobufjs () { return require('../../../datadog-plugin-protobufjs/src') }, get redis () { return require('../../../datadog-plugin-redis/src') }, get restify () { return require('../../../datadog-plugin-restify/src') }, get rhea () { return require('../../../datadog-plugin-rhea/src') }, diff --git a/packages/dd-trace/src/plugins/schema.js b/packages/dd-trace/src/plugins/schema.js new file mode 100644 index 00000000000..675ba6a715f --- /dev/null +++ b/packages/dd-trace/src/plugins/schema.js @@ -0,0 +1,35 @@ +'use strict' + +const Plugin = require('./plugin') + +const SERIALIZATION = 'serialization' +const DESERIALIZATION = 'deserialization' + +class SchemaPlugin extends Plugin { + constructor (...args) { + super(...args) + + this.addSub(`apm:${this.constructor.id}:serialize-start`, this.handleSerializeStart.bind(this)) + this.addSub(`apm:${this.constructor.id}:deserialize-end`, this.handleDeserializeFinish.bind(this)) + } + + handleSerializeStart (args) { + const activeSpan = this.tracer.scope().active() + if (activeSpan && this.config.dsmEnabled) { + this.constructor.schemaExtractor.attachSchemaOnSpan( + args, activeSpan, SERIALIZATION, this.tracer + ) + } + } + + handleDeserializeFinish (args) { + const activeSpan = this.tracer.scope().active() + if (activeSpan && this.config.dsmEnabled) { + this.constructor.schemaExtractor.attachSchemaOnSpan( + args, activeSpan, DESERIALIZATION, this.tracer + ) + } + } +} + +module.exports = SchemaPlugin diff --git a/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js b/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js index db602ef83aa..134724b593a 100644 --- a/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js +++ b/packages/dd-trace/test/datastreams/schemas/schema_builder.spec.js @@ -24,7 +24,7 @@ describe('SchemaBuilder', () => { const shouldExtractAddress = builder.shouldExtractSchema('address', 1) const shouldExtractPerson2 = builder.shouldExtractSchema('person', 0) const shouldExtractTooDeep = builder.shouldExtractSchema('city', 11) - const schema = builder.build() + const schema = SchemaBuilder.getSchemaDefinition(builder.build()) const expectedSchema = { components: { From bba5f3ddb3e6d693aa74584f8c73d5b5cbf4ea95 Mon Sep 17 00:00:00 2001 From: William Conti <58711692+wconti27@users.noreply.github.com> Date: Fri, 4 Oct 2024 12:38:04 -0400 Subject: [PATCH 19/27] feat(dsm): implement avro schemas for avsc package (#4726) * add avro (avsc) schemas support for DSM --- .github/workflows/plugins.yml | 9 + docs/API.md | 2 + docs/add-redirects.sh | 1 + docs/test.ts | 1 + index.d.ts | 7 + packages/datadog-instrumentations/src/avsc.js | 37 ++++ .../src/helpers/hooks.js | 1 + packages/datadog-plugin-avsc/src/index.js | 9 + .../src/schema_iterator.js | 169 +++++++++++++++++ packages/datadog-plugin-avsc/test/helpers.js | 31 +++ .../datadog-plugin-avsc/test/index.spec.js | 176 ++++++++++++++++++ .../test/schemas/advanced_user.avsc | 74 ++++++++ .../test/schemas/advanced_users.avro | Bin 0 -> 126 bytes .../expected_advanced_user_schema.json | 57 ++++++ .../test/schemas/expected_user_schema.json | 21 +++ .../test/schemas/user.avsc | 25 +++ .../test/schemas/users.avro | Bin 0 -> 11 bytes .../src/schema_iterator.js | 11 +- packages/dd-trace/src/plugins/index.js | 1 + 19 files changed, 625 insertions(+), 7 deletions(-) create mode 100644 packages/datadog-instrumentations/src/avsc.js create mode 100644 packages/datadog-plugin-avsc/src/index.js create mode 100644 packages/datadog-plugin-avsc/src/schema_iterator.js create mode 100644 packages/datadog-plugin-avsc/test/helpers.js create mode 100644 packages/datadog-plugin-avsc/test/index.spec.js create mode 100644 packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc create mode 100644 packages/datadog-plugin-avsc/test/schemas/advanced_users.avro create mode 100644 packages/datadog-plugin-avsc/test/schemas/expected_advanced_user_schema.json create mode 100644 packages/datadog-plugin-avsc/test/schemas/expected_user_schema.json create mode 100644 packages/datadog-plugin-avsc/test/schemas/user.avsc create mode 100644 packages/datadog-plugin-avsc/test/schemas/users.avro diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 60872f79740..ca9842944d1 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -136,6 +136,15 @@ jobs: - uses: actions/checkout@v4 - uses: ./.github/actions/plugins/test-and-upstream + avsc: + runs-on: ubuntu-latest + env: + PLUGINS: avsc + DD_DATA_STREAMS_ENABLED: true + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/plugins/test-and-upstream + aws-sdk: strategy: matrix: diff --git a/docs/API.md b/docs/API.md index 713c17032cf..271425c4f28 100644 --- a/docs/API.md +++ b/docs/API.md @@ -24,6 +24,7 @@ tracer.use('pg', {
+
@@ -102,6 +103,7 @@ tracer.use('pg', { * [amqp10](./interfaces/export_.plugins.amqp10.html) * [amqplib](./interfaces/export_.plugins.amqplib.html) +* [avsc](./interfaces/export_.plugins.avsc.html) * [aws-sdk](./interfaces/export_.plugins.aws_sdk.html) * [bluebird](./interfaces/export_.plugins.bluebird.html) * [couchbase](./interfaces/export_.plugins.couchbase.html) diff --git a/docs/add-redirects.sh b/docs/add-redirects.sh index 732c8a83607..92d58ba3263 100755 --- a/docs/add-redirects.sh +++ b/docs/add-redirects.sh @@ -14,6 +14,7 @@ echo "writing redirects..." declare -a plugins=( "amqp10" "amqplib" + "avsc" "aws_sdk" "bluebird" "couchbase" diff --git a/docs/test.ts b/docs/test.ts index c45d0f3d515..e948e4ff4dd 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -281,6 +281,7 @@ const openSearchOptions: plugins.opensearch = { tracer.use('amqp10'); tracer.use('amqplib'); +tracer.use('avsc'); tracer.use('aws-sdk'); tracer.use('aws-sdk', awsSdkOptions); tracer.use('bunyan'); diff --git a/index.d.ts b/index.d.ts index 115b123e569..bc17ef2dad5 100644 --- a/index.d.ts +++ b/index.d.ts @@ -147,6 +147,7 @@ interface Plugins { "amqp10": tracer.plugins.amqp10; "amqplib": tracer.plugins.amqplib; "apollo": tracer.plugins.apollo; + "avsc": tracer.plugins.avsc; "aws-sdk": tracer.plugins.aws_sdk; "bunyan": tracer.plugins.bunyan; "cassandra-driver": tracer.plugins.cassandra_driver; @@ -1192,6 +1193,12 @@ declare namespace tracer { signature?: boolean; } + /** + * This plugin automatically patches the [avsc](https://github.com/mtth/avsc) module + * to collect avro message schemas when Datastreams Monitoring is enabled. + */ + interface avsc extends Integration {} + /** * This plugin automatically instruments the * [aws-sdk](https://github.com/aws/aws-sdk-js) module. diff --git a/packages/datadog-instrumentations/src/avsc.js b/packages/datadog-instrumentations/src/avsc.js new file mode 100644 index 00000000000..6d71b1744bf --- /dev/null +++ b/packages/datadog-instrumentations/src/avsc.js @@ -0,0 +1,37 @@ +const shimmer = require('../../datadog-shimmer') +const { addHook } = require('./helpers/instrument') + +const dc = require('dc-polyfill') +const serializeChannel = dc.channel('apm:avsc:serialize-start') +const deserializeChannel = dc.channel('apm:avsc:deserialize-end') + +function wrapSerialization (Type) { + shimmer.wrap(Type.prototype, 'toBuffer', original => function () { + if (!serializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + serializeChannel.publish({ messageClass: this }) + return original.apply(this, arguments) + }) +} + +function wrapDeserialization (Type) { + shimmer.wrap(Type.prototype, 'fromBuffer', original => function () { + if (!deserializeChannel.hasSubscribers) { + return original.apply(this, arguments) + } + const result = original.apply(this, arguments) + deserializeChannel.publish({ messageClass: result }) + return result + }) +} + +addHook({ + name: 'avsc', + versions: ['>=5.0.0'] +}, avro => { + wrapDeserialization(avro.Type) + wrapSerialization(avro.Type) + + return avro +}) diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index eb2cbcb794c..693ad84abf6 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -27,6 +27,7 @@ module.exports = { aerospike: () => require('../aerospike'), amqp10: () => require('../amqp10'), amqplib: () => require('../amqplib'), + avsc: () => require('../avsc'), 'aws-sdk': () => require('../aws-sdk'), bluebird: () => require('../bluebird'), 'body-parser': () => require('../body-parser'), diff --git a/packages/datadog-plugin-avsc/src/index.js b/packages/datadog-plugin-avsc/src/index.js new file mode 100644 index 00000000000..be0ef970e50 --- /dev/null +++ b/packages/datadog-plugin-avsc/src/index.js @@ -0,0 +1,9 @@ +const SchemaPlugin = require('../../dd-trace/src/plugins/schema') +const SchemaExtractor = require('./schema_iterator') + +class AvscPlugin extends SchemaPlugin { + static get id () { return 'avsc' } + static get schemaExtractor () { return SchemaExtractor } +} + +module.exports = AvscPlugin diff --git a/packages/datadog-plugin-avsc/src/schema_iterator.js b/packages/datadog-plugin-avsc/src/schema_iterator.js new file mode 100644 index 00000000000..c748bbf9e75 --- /dev/null +++ b/packages/datadog-plugin-avsc/src/schema_iterator.js @@ -0,0 +1,169 @@ +const AVRO = 'avro' +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const log = require('../../dd-trace/src/log') +const { + SchemaBuilder +} = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +class SchemaExtractor { + constructor (schema) { + this.schema = schema + } + + static getType (type) { + const typeMapping = { + string: 'string', + int: 'integer', + long: 'integer', + float: 'number', + double: 'number', + boolean: 'boolean', + bytes: 'string', + record: 'object', + enum: 'string', + array: 'array', + map: 'object', + fixed: 'string' + } + const typeName = type.typeName ?? type.name ?? type + return typeName === 'null' ? typeName : typeMapping[typeName] || 'string' + } + + static extractProperty (field, schemaName, fieldName, builder, depth) { + let array = false + let type + let format + let enumValues + let description + let ref + + const fieldType = field.type?.types ?? field.type?.typeName ?? field.type + + if (Array.isArray(fieldType)) { + // Union Type + type = 'union[' + fieldType.map(t => SchemaExtractor.getType(t.type || t)).join(',') + ']' + } else if (fieldType === 'array') { + // Array Type + array = true + const nestedType = field.type.itemsType.typeName + type = SchemaExtractor.getType(nestedType) + } else if (fieldType === 'record') { + // Nested Record Type + type = 'object' + ref = `#/components/schemas/${field.type.name}` + if (!SchemaExtractor.extractSchema(field.type, builder, depth + 1, this)) { + return false + } + } else if (fieldType === 'enum') { + enumValues = [] + let i = 0 + type = 'string' + while (field.type.symbols[i]) { + enumValues.push(field.type.symbols[i]) + i += 1 + } + } else { + // Primitive type + type = SchemaExtractor.getType(fieldType.type || fieldType) + if (fieldType === 'bytes') { + format = 'byte' + } else if (fieldType === 'int') { + format = 'int32' + } else if (fieldType === 'long') { + format = 'int64' + } else if (fieldType === 'float') { + format = 'float' + } else if (fieldType === 'double') { + format = 'double' + } + } + + return builder.addProperty(schemaName, fieldName, array, type, description, ref, format, enumValues) + } + + static extractSchema (schema, builder, depth, extractor) { + depth += 1 + const schemaName = schema.name + if (extractor) { + // if we already have a defined extractor, this is a nested schema. create a new extractor for the nested + // schema, ensure it is added to our schema builder's cache, and replace the builders iterator with our + // nested schema iterator / extractor. Once complete, add the new schema to our builder's schemas. + const nestedSchemaExtractor = new SchemaExtractor(schema) + builder.iterator = nestedSchemaExtractor + const nestedSchema = SchemaBuilder.getSchema(schemaName, nestedSchemaExtractor, builder) + for (const nestedSubSchemaName in nestedSchema.components.schemas) { + if (nestedSchema.components.schemas.hasOwnProperty(nestedSubSchemaName)) { + builder.schema.components.schemas[nestedSubSchemaName] = nestedSchema.components.schemas[nestedSubSchemaName] + } + } + return true + } else { + if (!builder.shouldExtractSchema(schemaName, depth)) { + return false + } + for (const field of schema.fields) { + if (!this.extractProperty(field, schemaName, field.name, builder, depth)) { + log.warn(`DSM: Unable to extract field with name: ${field.name} from Avro schema with name: ${schemaName}`) + } + } + } + return true + } + + static extractSchemas (descriptor, dataStreamsProcessor) { + return dataStreamsProcessor.getSchema(descriptor.name, new SchemaExtractor(descriptor)) + } + + iterateOverSchema (builder) { + this.constructor.extractSchema(this.schema, builder, 0) + } + + static attachSchemaOnSpan (args, span, operation, tracer) { + const { messageClass } = args + const descriptor = messageClass?.constructor?.type ?? messageClass + + if (!descriptor || !span) { + return + } + + if (span.context()._tags[SCHEMA_TYPE] && operation === 'serialization') { + // we have already added a schema to this span, this call is an encode of nested schema types + return + } + + span.setTag(SCHEMA_TYPE, AVRO) + span.setTag(SCHEMA_NAME, descriptor.name) + span.setTag(SCHEMA_OPERATION, operation) + + if (!tracer._dataStreamsProcessor.canSampleSchema(operation)) { + return + } + + // if the span is unsampled, do not sample the schema + if (!tracer._prioritySampler.isSampled(span)) { + return + } + + const weight = tracer._dataStreamsProcessor.trySampleSchema(operation) + if (weight === 0) { + return + } + + const schemaData = SchemaBuilder.getSchemaDefinition( + this.extractSchemas(descriptor, tracer._dataStreamsProcessor) + ) + + span.setTag(SCHEMA_DEFINITION, schemaData.definition) + span.setTag(SCHEMA_WEIGHT, weight) + span.setTag(SCHEMA_ID, schemaData.id) + } +} + +module.exports = SchemaExtractor diff --git a/packages/datadog-plugin-avsc/test/helpers.js b/packages/datadog-plugin-avsc/test/helpers.js new file mode 100644 index 00000000000..8e5be7ac433 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/helpers.js @@ -0,0 +1,31 @@ +const fs = require('fs') + +async function loadMessage (avro, messageTypeName) { + if (messageTypeName === 'User') { + // Read and parse the Avro schema + const schema = JSON.parse(fs.readFileSync('packages/datadog-plugin-avsc/test/schemas/user.avsc', 'utf8')) + + // Create a file and write Avro data + const filePath = 'packages/datadog-plugin-avsc/test/schemas/users.avro' + + return { + schema, + path: filePath + } + } else if (messageTypeName === 'AdvancedUser') { + // Read and parse the Avro schema + const schema = JSON.parse(fs.readFileSync('packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc', 'utf8')) + + // Create a file and write Avro data + const filePath = 'packages/datadog-plugin-avsc/test/schemas/advanced_users.avro' + + return { + schema, + path: filePath + } + } +} + +module.exports = { + loadMessage +} diff --git a/packages/datadog-plugin-avsc/test/index.spec.js b/packages/datadog-plugin-avsc/test/index.spec.js new file mode 100644 index 00000000000..b3a6db0c1f1 --- /dev/null +++ b/packages/datadog-plugin-avsc/test/index.spec.js @@ -0,0 +1,176 @@ +'use strict' + +const fs = require('fs') +const { expect } = require('chai') +const agent = require('../../dd-trace/test/plugins/agent') +const path = require('path') +const { + SCHEMA_DEFINITION, + SCHEMA_ID, + SCHEMA_NAME, + SCHEMA_OPERATION, + SCHEMA_WEIGHT, + SCHEMA_TYPE +} = require('../../dd-trace/src/constants') +const sinon = require('sinon') +const { loadMessage } = require('./helpers') +const { SchemaBuilder } = require('../../dd-trace/src/datastreams/schemas/schema_builder') + +const BASIC_USER_SCHEMA_DEF = JSON.parse( + fs.readFileSync(path.join(__dirname, 'schemas/expected_user_schema.json'), 'utf8') +) +const ADVANCED_USER_SCHEMA_DEF = JSON.parse( + fs.readFileSync(path.join(__dirname, 'schemas/expected_advanced_user_schema.json'), 'utf8') +) + +const BASIC_USER_SCHEMA_ID = '1605040621379664412' +const ADVANCED_USER_SCHEMA_ID = '919692610494986520' + +function compareJson (expected, span) { + const actual = JSON.parse(span.context()._tags[SCHEMA_DEFINITION]) + return JSON.stringify(actual) === JSON.stringify(expected) +} + +describe('Plugin', () => { + describe('avsc', function () { + this.timeout(0) + let tracer + let avro + let dateNowStub + let mockTime = 0 + + withVersions('avsc', ['avsc'], (version) => { + before(() => { + tracer = require('../../dd-trace').init() + // reset sampled schemas + if (tracer._dataStreamsProcessor?._schemaSamplers) { + tracer._dataStreamsProcessor._schemaSamplers = [] + } + }) + + describe('without configuration', () => { + before(() => { + dateNowStub = sinon.stub(Date, 'now').callsFake(() => { + const returnValue = mockTime + mockTime += 50000 // Increment by 50000 ms to ensure each DSM schema is sampled + return returnValue + }) + const cache = SchemaBuilder.getCache() + cache.clear() + return agent.load('avsc').then(() => { + avro = require(`../../../versions/avsc@${version}`).get() + }) + }) + + after(() => { + dateNowStub.restore() + return agent.close({ ritmReset: false }) + }) + + it('should serialize basic schema correctly', async () => { + const loaded = await loadMessage(avro, 'User') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + + tracer.trace('user.serialize', span => { + const buf = type.toBuffer({ name: 'Alyssa', favorite_number: 256, favorite_color: null }) + fs.writeFileSync(filePath, buf) + + expect(span._name).to.equal('user.serialize') + + expect(compareJson(BASIC_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.User') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, BASIC_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should serialize the advanced schema correctly', async () => { + const loaded = await loadMessage(avro, 'AdvancedUser') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + + tracer.trace('advanced_user.serialize', span => { + const buf = type.toBuffer({ + name: 'Alyssa', + age: 30, + email: 'alyssa@example.com', + height: 5.6, + preferences: { theme: 'dark', notifications: 'enabled' }, + tags: ['vip', 'premium'], + status: 'ACTIVE', + profile_picture: Buffer.from('binarydata'), + metadata: Buffer.from('metadata12345678'), + address: { street: '123 Main St', city: 'Metropolis', zipcode: '12345' } + }) + fs.writeFileSync(filePath, buf) + + expect(span._name).to.equal('advanced_user.serialize') + + expect(compareJson(ADVANCED_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.AdvancedUser') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'serialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ADVANCED_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize basic schema correctly', async () => { + const loaded = await loadMessage(avro, 'User') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + const buf = type.toBuffer({ name: 'Alyssa', favorite_number: 256, favorite_color: null }) + fs.writeFileSync(filePath, buf) + + tracer.trace('user.deserialize', span => { + type.fromBuffer(buf) + + expect(span._name).to.equal('user.deserialize') + + expect(compareJson(BASIC_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.User') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, BASIC_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + + it('should deserialize advanced schema correctly', async () => { + const loaded = await loadMessage(avro, 'AdvancedUser') + const type = avro.parse(loaded.schema) + const filePath = loaded.path + const buf = type.toBuffer({ + name: 'Alyssa', + age: 30, + email: 'alyssa@example.com', + height: 5.6, + preferences: { theme: 'dark', notifications: 'enabled' }, + tags: ['vip', 'premium'], + status: 'ACTIVE', + profile_picture: Buffer.from('binarydata'), + metadata: Buffer.from('metadata12345678'), + address: { street: '123 Main St', city: 'Metropolis', zipcode: '12345' } + }) + fs.writeFileSync(filePath, buf) + + tracer.trace('advanced_user.deserialize', span => { + type.fromBuffer(buf) + + expect(span._name).to.equal('advanced_user.deserialize') + + expect(compareJson(ADVANCED_USER_SCHEMA_DEF, span)).to.equal(true) + expect(span.context()._tags).to.have.property(SCHEMA_TYPE, 'avro') + expect(span.context()._tags).to.have.property(SCHEMA_NAME, 'example.avro.AdvancedUser') + expect(span.context()._tags).to.have.property(SCHEMA_OPERATION, 'deserialization') + expect(span.context()._tags).to.have.property(SCHEMA_ID, ADVANCED_USER_SCHEMA_ID) + expect(span.context()._tags).to.have.property(SCHEMA_WEIGHT, 1) + }) + }) + }) + }) + }) +}) diff --git a/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc b/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc new file mode 100644 index 00000000000..c25081c495e --- /dev/null +++ b/packages/datadog-plugin-avsc/test/schemas/advanced_user.avsc @@ -0,0 +1,74 @@ +{ + "namespace": "example.avro", + "type": "record", + "name": "AdvancedUser", + "fields": [ + { + "name": "email", + "type": [ + "null", + "string" + ], + "default": null + }, + { + "name": "preferences", + "type": { + "type": "map", + "values": "string" + } + }, + { + "name": "tags", + "type": { + "type": "array", + "items": "string" + } + }, + { + "name": "status", + "type": { + "type": "enum", + "name": "Status", + "symbols": [ + "ACTIVE", + "INACTIVE", + "BANNED" + ] + } + }, + { + "name": "profile_picture", + "type": "bytes" + }, + { + "name": "metadata", + "type": { + "type": "fixed", + "name": "Metadata", + "size": 16 + } + }, + { + "name": "address", + "type": { + "type": "record", + "name": "Address", + "fields": [ + { + "name": "street", + "type": "string" + }, + { + "name": "city", + "type": "string" + }, + { + "name": "zipcode", + "type": "string" + } + ] + } + } + ] +} \ No newline at end of file diff --git a/packages/datadog-plugin-avsc/test/schemas/advanced_users.avro b/packages/datadog-plugin-avsc/test/schemas/advanced_users.avro new file mode 100644 index 0000000000000000000000000000000000000000..1e31871c28e2a84340cd2a2542f9e32f52d3724c GIT binary patch literal 126 zcmW;By9&ZU5JureLXZ@xLJF%C)`E9SUm%rF@FzRske%IScNFvXUaIrKS*+paoN=W$ zBs$uRL5fNrB($_h)5-_sJC9HfKG) Date: Mon, 7 Oct 2024 14:31:53 -0400 Subject: [PATCH 20/27] use AsyncLocalStorage instead of our home-grown solutions (#4201) * use AsyncLocalStorage instead of our home-grown solutions The comment in the file that selected a storage implementation suggested just using AsyncLocalStorage once it supports triggerAsyncResource(). That said, literally zero of our code uses triggerAsyncResource(), so this is assumed to be historical and no longer relevant. Switching to stock AsyncLocalStorage will enable the usage of TracingChannel in the future. * self-contain profiling's AsyncLocalStorage channel usage * remove flag detection --- packages/datadog-core/index.js | 4 +- .../src/storage/async_resource.js | 108 ------------ packages/datadog-core/src/storage/index.js | 5 - packages/datadog-core/test/setup.js | 8 - .../test/storage/async_resource.spec.js | 20 --- packages/datadog-core/test/storage/test.js | 160 ------------------ .../dd-trace/src/profiling/profilers/wall.js | 40 +++++ 7 files changed, 42 insertions(+), 303 deletions(-) delete mode 100644 packages/datadog-core/src/storage/async_resource.js delete mode 100644 packages/datadog-core/src/storage/index.js delete mode 100644 packages/datadog-core/test/setup.js delete mode 100644 packages/datadog-core/test/storage/async_resource.spec.js delete mode 100644 packages/datadog-core/test/storage/test.js diff --git a/packages/datadog-core/index.js b/packages/datadog-core/index.js index 72b0403aa75..9819b32f3ba 100644 --- a/packages/datadog-core/index.js +++ b/packages/datadog-core/index.js @@ -1,7 +1,7 @@ 'use strict' -const LocalStorage = require('./src/storage') +const { AsyncLocalStorage } = require('async_hooks') -const storage = new LocalStorage() +const storage = new AsyncLocalStorage() module.exports = { storage } diff --git a/packages/datadog-core/src/storage/async_resource.js b/packages/datadog-core/src/storage/async_resource.js deleted file mode 100644 index 4738845e415..00000000000 --- a/packages/datadog-core/src/storage/async_resource.js +++ /dev/null @@ -1,108 +0,0 @@ -'use strict' - -const { createHook, executionAsyncResource } = require('async_hooks') -const { channel } = require('dc-polyfill') - -const beforeCh = channel('dd-trace:storage:before') -const afterCh = channel('dd-trace:storage:after') -const enterCh = channel('dd-trace:storage:enter') - -let PrivateSymbol = Symbol -function makePrivateSymbol () { - // eslint-disable-next-line no-new-func - PrivateSymbol = new Function('name', 'return %CreatePrivateSymbol(name)') -} - -try { - makePrivateSymbol() -} catch (e) { - try { - const v8 = require('v8') - v8.setFlagsFromString('--allow-natives-syntax') - makePrivateSymbol() - v8.setFlagsFromString('--no-allow-natives-syntax') - // eslint-disable-next-line no-empty - } catch (e) {} -} - -class AsyncResourceStorage { - constructor () { - this._ddResourceStore = PrivateSymbol('ddResourceStore') - this._enabled = false - this._hook = createHook(this._createHook()) - } - - disable () { - if (!this._enabled) return - - this._hook.disable() - this._enabled = false - } - - getStore () { - if (!this._enabled) return - - const resource = this._executionAsyncResource() - - return resource[this._ddResourceStore] - } - - enterWith (store) { - this._enable() - - const resource = this._executionAsyncResource() - - resource[this._ddResourceStore] = store - enterCh.publish() - } - - run (store, callback, ...args) { - this._enable() - - const resource = this._executionAsyncResource() - const oldStore = resource[this._ddResourceStore] - - resource[this._ddResourceStore] = store - enterCh.publish() - - try { - return callback(...args) - } finally { - resource[this._ddResourceStore] = oldStore - enterCh.publish() - } - } - - _createHook () { - return { - init: this._init.bind(this), - before () { - beforeCh.publish() - }, - after () { - afterCh.publish() - } - } - } - - _enable () { - if (this._enabled) return - - this._enabled = true - this._hook.enable() - } - - _init (asyncId, type, triggerAsyncId, resource) { - const currentResource = this._executionAsyncResource() - - if (Object.prototype.hasOwnProperty.call(currentResource, this._ddResourceStore)) { - resource[this._ddResourceStore] = currentResource[this._ddResourceStore] - } - } - - _executionAsyncResource () { - return executionAsyncResource() || {} - } -} - -module.exports = AsyncResourceStorage diff --git a/packages/datadog-core/src/storage/index.js b/packages/datadog-core/src/storage/index.js deleted file mode 100644 index e522e61ced2..00000000000 --- a/packages/datadog-core/src/storage/index.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict' - -// TODO: default to AsyncLocalStorage when it supports triggerAsyncResource - -module.exports = require('./async_resource') diff --git a/packages/datadog-core/test/setup.js b/packages/datadog-core/test/setup.js deleted file mode 100644 index 2f8af45cdd2..00000000000 --- a/packages/datadog-core/test/setup.js +++ /dev/null @@ -1,8 +0,0 @@ -'use strict' - -require('tap').mochaGlobals() - -const chai = require('chai') -const sinonChai = require('sinon-chai') - -chai.use(sinonChai) diff --git a/packages/datadog-core/test/storage/async_resource.spec.js b/packages/datadog-core/test/storage/async_resource.spec.js deleted file mode 100644 index ce19b216260..00000000000 --- a/packages/datadog-core/test/storage/async_resource.spec.js +++ /dev/null @@ -1,20 +0,0 @@ -'use strict' - -require('../setup') - -const StorageBackend = require('../../src/storage/async_resource') -const testStorage = require('./test') - -describe('storage/async_resource', () => { - let storage - - beforeEach(() => { - storage = new StorageBackend() - }) - - afterEach(() => { - storage.disable() - }) - - testStorage(() => storage) -}) diff --git a/packages/datadog-core/test/storage/test.js b/packages/datadog-core/test/storage/test.js deleted file mode 100644 index 0f69a43d9f0..00000000000 --- a/packages/datadog-core/test/storage/test.js +++ /dev/null @@ -1,160 +0,0 @@ -'use strict' - -const { expect } = require('chai') -const { inspect } = require('util') -const { - AsyncResource, - executionAsyncId, - executionAsyncResource -} = require('async_hooks') - -module.exports = factory => { - let storage - let store - - beforeEach(() => { - storage = factory() - store = {} - }) - - describe('getStore()', () => { - it('should return undefined by default', () => { - expect(storage.getStore()).to.be.undefined - }) - }) - - describe('run()', () => { - it('should return the value returned by the callback', () => { - expect(storage.run(store, () => 'test')).to.equal('test') - }) - - it('should preserve the surrounding scope', () => { - expect(storage.getStore()).to.be.undefined - - storage.run(store, () => {}) - - expect(storage.getStore()).to.be.undefined - }) - - it('should run the span on the current scope', () => { - expect(storage.getStore()).to.be.undefined - - storage.run(store, () => { - expect(storage.getStore()).to.equal(store) - }) - - expect(storage.getStore()).to.be.undefined - }) - - it('should persist through setTimeout', done => { - storage.run(store, () => { - setTimeout(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through setImmediate', done => { - storage.run(store, () => { - setImmediate(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through setInterval', done => { - storage.run(store, () => { - let shouldReturn = false - - const timer = setInterval(() => { - expect(storage.getStore()).to.equal(store) - - if (shouldReturn) { - clearInterval(timer) - return done() - } - - shouldReturn = true - }, 0) - }) - }) - - it('should persist through process.nextTick', done => { - storage.run(store, () => { - process.nextTick(() => { - expect(storage.getStore()).to.equal(store) - done() - }, 0) - }) - }) - - it('should persist through promises', () => { - const promise = Promise.resolve() - - return storage.run(store, () => { - return promise.then(() => { - expect(storage.getStore()).to.equal(store) - }) - }) - }) - - it('should handle concurrency', done => { - storage.run(store, () => { - setImmediate(() => { - expect(storage.getStore()).to.equal(store) - done() - }) - }) - - storage.run(store, () => {}) - }) - - it('should not break propagation for nested resources', done => { - storage.run(store, () => { - const asyncResource = new AsyncResource( - 'TEST', { triggerAsyncId: executionAsyncId(), requireManualDestroy: false } - ) - - asyncResource.runInAsyncScope(() => {}) - - expect(storage.getStore()).to.equal(store) - - done() - }) - }) - - it('should not log ddResourceStore contents', done => { - function getKeys (output) { - return output.split('\n').slice(1, -1).map(line => { - return line.split(':').map(v => v.trim())[0] - }) - } - - setImmediate(() => { - const withoutStore = getKeys(inspect(executionAsyncResource(), { depth: 0 })) - storage.run(store, () => { - setImmediate(() => { - const withStore = getKeys(inspect(executionAsyncResource(), { depth: 0 })) - expect(withStore).to.deep.equal(withoutStore) - done() - }) - }) - }) - }) - }) - - describe('enterWith()', () => { - it('should transition into the context for the remainder of the current execution', () => { - const newStore = {} - - storage.run(store, () => { - storage.enterWith(newStore) - expect(storage.getStore()).to.equal(newStore) - }) - - expect(storage.getStore()).to.be.undefined - }) - }) -} diff --git a/packages/dd-trace/src/profiling/profilers/wall.js b/packages/dd-trace/src/profiling/profilers/wall.js index ee23b1145b0..39af4ca2bfc 100644 --- a/packages/dd-trace/src/profiling/profilers/wall.js +++ b/packages/dd-trace/src/profiling/profilers/wall.js @@ -76,6 +76,44 @@ function getWebTags (startedSpans, i, span) { return memoize(null) } +let channelsActivated = false +function ensureChannelsActivated () { + if (channelsActivated) return + + const { AsyncLocalStorage, createHook } = require('async_hooks') + const shimmer = require('../../../../datadog-shimmer') + + createHook({ before: () => beforeCh.publish() }).enable() + + let inRun = false + shimmer.wrap(AsyncLocalStorage.prototype, 'enterWith', function (original) { + return function (...args) { + const retVal = original.apply(this, args) + if (!inRun) enterCh.publish() + return retVal + } + }) + + shimmer.wrap(AsyncLocalStorage.prototype, 'run', function (original) { + return function (store, callback, ...args) { + const wrappedCb = shimmer.wrapFunction(callback, cb => function (...args) { + inRun = false + enterCh.publish() + const retVal = cb.apply(this, args) + inRun = true + return retVal + }) + inRun = true + const retVal = original.call(this, store, wrappedCb, ...args) + enterCh.publish() + inRun = false + return retVal + } + }) + + channelsActivated = true +} + class NativeWallProfiler { constructor (options = {}) { this.type = 'wall' @@ -121,6 +159,8 @@ class NativeWallProfiler { start ({ mapper } = {}) { if (this._started) return + ensureChannelsActivated() + this._mapper = mapper this._pprof = require('@datadog/pprof') kSampleCount = this._pprof.time.constants.kSampleCount From a11a1fd20ea911044bd68eb952b56abce21d6693 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 8 Oct 2024 09:34:30 +0200 Subject: [PATCH 21/27] Upgrade iast rewriter to 2.5.0 (#4761) * Upgrade iast rewriter version to 2.5.0 * Implement tplOperator tracking method --- package.json | 2 +- .../src/appsec/iast/taint-tracking/csi-methods.js | 1 + .../iast/taint-tracking/taint-tracking-impl.js | 15 +++++++++++++++ .../resources/propagationFunctions.js | 8 ++++++++ .../taint-tracking/taint-tracking-impl.spec.js | 4 +++- yarn.lock | 8 ++++---- 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 821ed481d9a..a785ea6314e 100644 --- a/package.json +++ b/package.json @@ -77,7 +77,7 @@ }, "dependencies": { "@datadog/native-appsec": "8.1.1", - "@datadog/native-iast-rewriter": "2.4.1", + "@datadog/native-iast-rewriter": "2.5.0", "@datadog/native-iast-taint-tracking": "3.1.0", "@datadog/native-metrics": "^2.0.0", "@datadog/pprof": "5.3.0", diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js b/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js index 62f49f2e830..2133971afb9 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/csi-methods.js @@ -12,6 +12,7 @@ const csiMethods = [ { src: 'substring' }, { src: 'toLowerCase', dst: 'stringCase' }, { src: 'toUpperCase', dst: 'stringCase' }, + { src: 'tplOperator', operator: true }, { src: 'trim' }, { src: 'trimEnd' }, { src: 'trimStart', dst: 'trim' }, diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js index 9f48a3add3f..5fa16d00d77 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js @@ -29,6 +29,7 @@ const TaintTrackingNoop = { substr: noop, substring: noop, stringCase: noop, + tplOperator: noop, trim: noop, trimEnd: noop } @@ -117,6 +118,20 @@ function csiMethodsOverrides (getContext) { return res }, + tplOperator: function (res, ...rest) { + try { + const iastContext = getContext() + const transactionId = getTransactionId(iastContext) + if (transactionId) { + return TaintedUtils.concat(transactionId, res, ...rest) + } + } catch (e) { + iastLog.error('Error invoking CSI tplOperator') + .errorAndPublish(e) + } + return res + }, + stringCase: getCsiFn( (transactionId, res, target) => TaintedUtils.stringCase(transactionId, res, target), getContext, diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js index 4028f265b3e..de37c351789 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/resources/propagationFunctions.js @@ -12,6 +12,13 @@ function templateLiteralEndingWithNumberParams (str) { return `${str}Literal${num1}${num2}` } +function templateLiteralWithTaintedAtTheEnd (str) { + const num1 = 1 + const num2 = 2 + const hello = 'world' + return `Literal${num1}${num2}-${hello}-${str}` +} + function appendStr (str) { let pre = 'pre_' pre += str @@ -108,6 +115,7 @@ module.exports = { substrStr, substringStr, templateLiteralEndingWithNumberParams, + templateLiteralWithTaintedAtTheEnd, toLowerCaseStr, toUpperCaseStr, trimEndStr, diff --git a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js index e0eb9fc580a..d356753d607 100644 --- a/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js +++ b/packages/dd-trace/test/appsec/iast/taint-tracking/taint-tracking-impl.spec.js @@ -26,6 +26,7 @@ const propagationFns = [ 'substrStr', 'substringStr', 'templateLiteralEndingWithNumberParams', + 'templateLiteralWithTaintedAtTheEnd', 'toLowerCaseStr', 'toUpperCaseStr', 'trimEndStr', @@ -137,7 +138,8 @@ describe('TaintTracking', () => { 'concatSuffix', 'concatTaintedStr', 'insertStr', - 'templateLiteralEndingWithNumberParams' + 'templateLiteralEndingWithNumberParams', + 'templateLiteralWithTaintedAtTheEnd' ] propagationFns.forEach((propFn) => { if (filtered.includes(propFn)) return diff --git a/yarn.lock b/yarn.lock index cf7cba3f3f4..62d059200d9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -263,10 +263,10 @@ dependencies: node-gyp-build "^3.9.0" -"@datadog/native-iast-rewriter@2.4.1": - version "2.4.1" - resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.4.1.tgz#e8211f78c818906513fb96a549374da0382c7623" - integrity sha512-j3auTmyyn63e2y+SL28CGNy/l+jXQyh+pxqoGTacWaY5FW/dvo5nGQepAismgJ3qJ8VhQfVWRdxBSiT7wu9clw== +"@datadog/native-iast-rewriter@2.5.0": + version "2.5.0" + resolved "https://registry.yarnpkg.com/@datadog/native-iast-rewriter/-/native-iast-rewriter-2.5.0.tgz#b613defe86e78168f750d1f1662d4ffb3cf002e6" + integrity sha512-WRu34A3Wwp6oafX8KWNAbedtDaaJO+nzfYQht7pcJKjyC2ggfPeF7SoP+eDo9wTn4/nQwEOscSR4hkJqTRlpXQ== dependencies: lru-cache "^7.14.0" node-gyp-build "^4.5.0" From 111a156693a43471e42f7482e29d9cea91738aef Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Tue, 8 Oct 2024 10:05:07 +0200 Subject: [PATCH 22/27] Exploit Prevention LFI (#4676) * rasp lfi and iast using rasp fs-plugin * Add rasp lfi capability in RC * Handle aborted operations in fs instrumentation * enable test without express * cleanup and console log to debug test error * Do not throw * another test * Try increasing timeout * Enable debug again * Enable debug again * increase timeout a lot * increase timeout more * New lfi test * Increase test timeout * print all errors * remote debug info * Handle the different invocation cases * Handle non string properties * specify types to be analyzed * a bunch of tests * clean up * rasp lfi subs delayed (#4715) * Delay Appsec fs plugin subscription to fs:operations until the first req is received * disable rasp in tests * fix tests recursive call * Avoid multiple subscriptions to incomingHttpRequestStart * another try * replace spy with stub * execute unsubscribe asynchronously * sinon.assert async * clarify comment * Use a constant * Do not enable rasp in some tests * Remove not needed config property * Rename properties * Test iast and rasp fs-plugin subscription order * Avoid multiple analyzeLfi subscriptions * Block synchronous operations * Include synchronous blocking integration test * Test refactor * rename test file * Cleanup --- .../datadog-instrumentations/src/express.js | 23 + packages/datadog-instrumentations/src/fs.js | 34 +- packages/dd-trace/src/appsec/addresses.js | 2 + packages/dd-trace/src/appsec/channels.js | 4 +- .../iast/analyzers/path-traversal-analyzer.js | 9 +- packages/dd-trace/src/appsec/iast/index.js | 3 + .../dd-trace/src/appsec/rasp/fs-plugin.js | 99 ++++ packages/dd-trace/src/appsec/rasp/index.js | 34 +- packages/dd-trace/src/appsec/rasp/lfi.js | 112 +++++ packages/dd-trace/src/appsec/rasp/utils.js | 3 +- .../src/appsec/remote_config/capabilities.js | 1 + .../src/appsec/remote_config/index.js | 2 + .../analyzers/path-traversal-analyzer.spec.js | 8 + .../dd-trace/test/appsec/iast/index.spec.js | 30 +- packages/dd-trace/test/appsec/index.spec.js | 5 +- .../test/appsec/rasp/fs-plugin.spec.js | 251 ++++++++++ .../dd-trace/test/appsec/rasp/index.spec.js | 73 ++- .../appsec/rasp/lfi.express.plugin.spec.js | 469 ++++++++++++++++++ .../lfi.integration.express.plugin.spec.js | 69 +++ .../dd-trace/test/appsec/rasp/lfi.spec.js | 144 ++++++ .../appsec/rasp/resources/lfi-app/index.js | 28 ++ .../appsec/rasp/resources/lfi_rasp_rules.json | 61 +++ packages/dd-trace/test/appsec/rasp/utils.js | 12 +- .../test/appsec/remote_config/index.spec.js | 10 + .../test/appsec/response_blocking.spec.js | 5 +- packages/dd-trace/test/plugins/agent.js | 4 +- 26 files changed, 1465 insertions(+), 30 deletions(-) create mode 100644 packages/dd-trace/src/appsec/rasp/fs-plugin.js create mode 100644 packages/dd-trace/src/appsec/rasp/lfi.js create mode 100644 packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/rasp/lfi.spec.js create mode 100644 packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js create mode 100644 packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json diff --git a/packages/datadog-instrumentations/src/express.js b/packages/datadog-instrumentations/src/express.js index d3113821364..c47feef2468 100644 --- a/packages/datadog-instrumentations/src/express.js +++ b/packages/datadog-instrumentations/src/express.js @@ -3,6 +3,7 @@ const { createWrapRouterMethod } = require('./router') const shimmer = require('../../datadog-shimmer') const { addHook, channel } = require('./helpers/instrument') +const tracingChannel = require('dc-polyfill').tracingChannel const handleChannel = channel('apm:express:request:handle') @@ -35,6 +36,27 @@ function wrapResponseJson (json) { } } +const responseRenderChannel = tracingChannel('datadog:express:response:render') + +function wrapResponseRender (render) { + return function wrappedRender (view, options, callback) { + if (!responseRenderChannel.start.hasSubscribers) { + return render.apply(this, arguments) + } + + return responseRenderChannel.traceSync( + render, + { + req: this.req, + view, + options + }, + this, + ...arguments + ) + } +} + addHook({ name: 'express', versions: ['>=4'] }, express => { shimmer.wrap(express.application, 'handle', wrapHandle) shimmer.wrap(express.Router, 'use', wrapRouterMethod) @@ -42,6 +64,7 @@ addHook({ name: 'express', versions: ['>=4'] }, express => { shimmer.wrap(express.response, 'json', wrapResponseJson) shimmer.wrap(express.response, 'jsonp', wrapResponseJson) + shimmer.wrap(express.response, 'render', wrapResponseRender) return express }) diff --git a/packages/datadog-instrumentations/src/fs.js b/packages/datadog-instrumentations/src/fs.js index e0e57f1ebce..9ae201b9860 100644 --- a/packages/datadog-instrumentations/src/fs.js +++ b/packages/datadog-instrumentations/src/fs.js @@ -266,24 +266,44 @@ function createWrapFunction (prefix = '', override = '') { const lastIndex = arguments.length - 1 const cb = typeof arguments[lastIndex] === 'function' && arguments[lastIndex] const innerResource = new AsyncResource('bound-anonymous-fn') - const message = getMessage(method, getMethodParamsRelationByPrefix(prefix)[operation], arguments, this) + const params = getMethodParamsRelationByPrefix(prefix)[operation] + const abortController = new AbortController() + const message = { ...getMessage(method, params, arguments, this), abortController } + + const finish = innerResource.bind(function (error) { + if (error !== null && typeof error === 'object') { // fs.exists receives a boolean + errorChannel.publish(error) + } + finishChannel.publish() + }) if (cb) { const outerResource = new AsyncResource('bound-anonymous-fn') arguments[lastIndex] = shimmer.wrapFunction(cb, cb => innerResource.bind(function (e) { - if (e !== null && typeof e === 'object') { // fs.exists receives a boolean - errorChannel.publish(e) - } - - finishChannel.publish() - + finish(e) return outerResource.runInAsyncScope(() => cb.apply(this, arguments)) })) } return innerResource.runInAsyncScope(() => { startChannel.publish(message) + + if (abortController.signal.aborted) { + const error = abortController.signal.reason || new Error('Aborted') + + if (prefix === 'promises.') { + finish(error) + return Promise.reject(error) + } else if (name.includes('Sync') || !cb) { + finish(error) + throw error + } else if (cb) { + arguments[lastIndex](error) + return + } + } + try { const result = original.apply(this, arguments) if (cb) return result diff --git a/packages/dd-trace/src/appsec/addresses.js b/packages/dd-trace/src/appsec/addresses.js index e2cf6c6940a..f8ce3033d36 100644 --- a/packages/dd-trace/src/appsec/addresses.js +++ b/packages/dd-trace/src/appsec/addresses.js @@ -23,6 +23,8 @@ module.exports = { WAF_CONTEXT_PROCESSOR: 'waf.context.processor', HTTP_OUTGOING_URL: 'server.io.net.url', + FS_OPERATION_PATH: 'server.io.fs.file', + DB_STATEMENT: 'server.db.statement', DB_SYSTEM: 'server.db.system' } diff --git a/packages/dd-trace/src/appsec/channels.js b/packages/dd-trace/src/appsec/channels.js index a451b9ce145..729f4da0334 100644 --- a/packages/dd-trace/src/appsec/channels.js +++ b/packages/dd-trace/src/appsec/channels.js @@ -25,5 +25,7 @@ module.exports = { pgQueryStart: dc.channel('apm:pg:query:start'), pgPoolQueryStart: dc.channel('datadog:pg:pool:query:start'), mysql2OuterQueryStart: dc.channel('datadog:mysql2:outerquery:start'), - wafRunFinished: dc.channel('datadog:waf:run:finish') + wafRunFinished: dc.channel('datadog:waf:run:finish'), + fsOperationStart: dc.channel('apm:fs:operation:start'), + expressMiddlewareError: dc.channel('apm:express:middleware:error') } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js index 83bf2a87085..625dbde9150 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/path-traversal-analyzer.js @@ -29,7 +29,14 @@ class PathTraversalAnalyzer extends InjectionAnalyzer { onConfigure () { this.addSub('apm:fs:operation:start', (obj) => { - if (ignoredOperations.includes(obj.operation)) return + const store = storage.getStore() + const outOfReqOrChild = !store?.fs?.root + + // we could filter out all the nested fs.operations based on store.fs.root + // but if we spect a store in the context to be present we are going to exclude + // all out_of_the_request fs.operations + // AppsecFsPlugin must be enabled + if (ignoredOperations.includes(obj.operation) || outOfReqOrChild) return const pathArguments = [] if (obj.dest) { diff --git a/packages/dd-trace/src/appsec/iast/index.js b/packages/dd-trace/src/appsec/iast/index.js index 0facaa39a2a..9330bfdbbb1 100644 --- a/packages/dd-trace/src/appsec/iast/index.js +++ b/packages/dd-trace/src/appsec/iast/index.js @@ -14,6 +14,7 @@ const { } = require('./taint-tracking') const { IAST_ENABLED_TAG_KEY } = require('./tags') const iastTelemetry = require('./telemetry') +const { enable: enableFsPlugin, disable: disableFsPlugin, IAST_MODULE } = require('../rasp/fs-plugin') // TODO Change to `apm:http:server:request:[start|close]` when the subscription // order of the callbacks can be enforce @@ -27,6 +28,7 @@ function enable (config, _tracer) { if (isEnabled) return iastTelemetry.configure(config, config.iast?.telemetryVerbosity) + enableFsPlugin(IAST_MODULE) enableAllAnalyzers(config) enableTaintTracking(config.iast, iastTelemetry.verbosity) requestStart.subscribe(onIncomingHttpRequestStart) @@ -44,6 +46,7 @@ function disable () { isEnabled = false iastTelemetry.stop() + disableFsPlugin(IAST_MODULE) disableAllAnalyzers() disableTaintTracking() overheadController.finishGlobalContext() diff --git a/packages/dd-trace/src/appsec/rasp/fs-plugin.js b/packages/dd-trace/src/appsec/rasp/fs-plugin.js new file mode 100644 index 00000000000..a283b4f1a61 --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/fs-plugin.js @@ -0,0 +1,99 @@ +'use strict' + +const Plugin = require('../../plugins/plugin') +const { storage } = require('../../../../datadog-core') +const log = require('../../log') + +const RASP_MODULE = 'rasp' +const IAST_MODULE = 'iast' + +const enabledFor = { + [RASP_MODULE]: false, + [IAST_MODULE]: false +} + +let fsPlugin + +function enterWith (fsProps, store = storage.getStore()) { + if (store && !store.fs?.opExcluded) { + storage.enterWith({ + ...store, + fs: { + ...store.fs, + ...fsProps, + parentStore: store + } + }) + } +} + +class AppsecFsPlugin extends Plugin { + enable () { + this.addSub('apm:fs:operation:start', this._onFsOperationStart) + this.addSub('apm:fs:operation:finish', this._onFsOperationFinishOrRenderEnd) + this.addSub('tracing:datadog:express:response:render:start', this._onResponseRenderStart) + this.addSub('tracing:datadog:express:response:render:end', this._onFsOperationFinishOrRenderEnd) + + super.configure(true) + } + + disable () { + super.configure(false) + } + + _onFsOperationStart () { + const store = storage.getStore() + if (store) { + enterWith({ root: store.fs?.root === undefined }, store) + } + } + + _onResponseRenderStart () { + enterWith({ opExcluded: true }) + } + + _onFsOperationFinishOrRenderEnd () { + const store = storage.getStore() + if (store?.fs?.parentStore) { + storage.enterWith(store.fs.parentStore) + } + } +} + +function enable (mod) { + if (enabledFor[mod] !== false) return + + enabledFor[mod] = true + + if (!fsPlugin) { + fsPlugin = new AppsecFsPlugin() + fsPlugin.enable() + } + + log.info(`Enabled AppsecFsPlugin for ${mod}`) +} + +function disable (mod) { + if (!mod || !enabledFor[mod]) return + + enabledFor[mod] = false + + const allDisabled = Object.values(enabledFor).every(val => val === false) + if (allDisabled) { + fsPlugin?.disable() + + fsPlugin = undefined + } + + log.info(`Disabled AppsecFsPlugin for ${mod}`) +} + +module.exports = { + enable, + disable, + + AppsecFsPlugin, + + RASP_MODULE, + IAST_MODULE +} diff --git a/packages/dd-trace/src/appsec/rasp/index.js b/packages/dd-trace/src/appsec/rasp/index.js index 801608e54d8..d5a1312872a 100644 --- a/packages/dd-trace/src/appsec/rasp/index.js +++ b/packages/dd-trace/src/appsec/rasp/index.js @@ -1,10 +1,11 @@ 'use strict' const web = require('../../plugins/util/web') -const { setUncaughtExceptionCaptureCallbackStart } = require('../channels') -const { block } = require('../blocking') +const { setUncaughtExceptionCaptureCallbackStart, expressMiddlewareError } = require('../channels') +const { block, isBlocked } = require('../blocking') const ssrf = require('./ssrf') const sqli = require('./sql_injection') +const lfi = require('./lfi') const { DatadogRaspAbortError } = require('./utils') @@ -30,17 +31,13 @@ function findDatadogRaspAbortError (err, deep = 10) { return err } - if (err.cause && deep > 0) { + if (err?.cause && deep > 0) { return findDatadogRaspAbortError(err.cause, deep - 1) } } -function handleUncaughtExceptionMonitor (err) { - const abortError = findDatadogRaspAbortError(err) - if (!abortError) return - - const { req, res, blockingAction } = abortError - block(req, res, web.root(req), null, blockingAction) +function handleUncaughtExceptionMonitor (error) { + if (!blockOnDatadogRaspAbortError({ error })) return if (!process.hasUncaughtExceptionCaptureCallback()) { const cleanUp = removeAllListeners(process, 'uncaughtException') @@ -82,22 +79,39 @@ function handleUncaughtExceptionMonitor (err) { } } +function blockOnDatadogRaspAbortError ({ error }) { + const abortError = findDatadogRaspAbortError(error) + if (!abortError) return false + + const { req, res, blockingAction } = abortError + if (!isBlocked(res)) { + block(req, res, web.root(req), null, blockingAction) + } + + return true +} + function enable (config) { ssrf.enable(config) sqli.enable(config) + lfi.enable(config) process.on('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor) + expressMiddlewareError.subscribe(blockOnDatadogRaspAbortError) } function disable () { ssrf.disable() sqli.disable() + lfi.disable() process.off('uncaughtExceptionMonitor', handleUncaughtExceptionMonitor) + if (expressMiddlewareError.hasSubscribers) expressMiddlewareError.unsubscribe(blockOnDatadogRaspAbortError) } module.exports = { enable, disable, - handleUncaughtExceptionMonitor // exported only for testing purpose + handleUncaughtExceptionMonitor, // exported only for testing purpose + blockOnDatadogRaspAbortError // exported only for testing purpose } diff --git a/packages/dd-trace/src/appsec/rasp/lfi.js b/packages/dd-trace/src/appsec/rasp/lfi.js new file mode 100644 index 00000000000..1190734064d --- /dev/null +++ b/packages/dd-trace/src/appsec/rasp/lfi.js @@ -0,0 +1,112 @@ +'use strict' + +const { fsOperationStart, incomingHttpRequestStart } = require('../channels') +const { storage } = require('../../../../datadog-core') +const { enable: enableFsPlugin, disable: disableFsPlugin, RASP_MODULE } = require('./fs-plugin') +const { FS_OPERATION_PATH } = require('../addresses') +const waf = require('../waf') +const { RULE_TYPES, handleResult } = require('./utils') +const { isAbsolute } = require('path') + +let config +let enabled +let analyzeSubscribed + +function enable (_config) { + config = _config + + if (enabled) return + + enabled = true + + incomingHttpRequestStart.subscribe(onFirstReceivedRequest) +} + +function disable () { + if (fsOperationStart.hasSubscribers) fsOperationStart.unsubscribe(analyzeLfi) + if (incomingHttpRequestStart.hasSubscribers) incomingHttpRequestStart.unsubscribe(onFirstReceivedRequest) + + disableFsPlugin(RASP_MODULE) + + enabled = false + analyzeSubscribed = false +} + +function onFirstReceivedRequest () { + // nodejs unsubscribe during publish bug: https://github.com/nodejs/node/pull/55116 + process.nextTick(() => { + incomingHttpRequestStart.unsubscribe(onFirstReceivedRequest) + }) + + enableFsPlugin(RASP_MODULE) + + if (!analyzeSubscribed) { + fsOperationStart.subscribe(analyzeLfi) + analyzeSubscribed = true + } +} + +function analyzeLfi (ctx) { + const store = storage.getStore() + if (!store) return + + const { req, fs, res } = store + if (!req || !fs) return + + getPaths(ctx, fs).forEach(path => { + const persistent = { + [FS_OPERATION_PATH]: path + } + + const result = waf.run({ persistent }, req, RULE_TYPES.LFI) + handleResult(result, req, res, ctx.abortController, config) + }) +} + +function getPaths (ctx, fs) { + // these properties could have String, Buffer, URL, Integer or FileHandle types + const pathArguments = [ + ctx.dest, + ctx.existingPath, + ctx.file, + ctx.newPath, + ctx.oldPath, + ctx.path, + ctx.prefix, + ctx.src, + ctx.target + ] + + return pathArguments + .map(path => pathToStr(path)) + .filter(path => shouldAnalyze(path, fs)) +} + +function pathToStr (path) { + if (!path) return + + if (typeof path === 'string' || + path instanceof String || + path instanceof Buffer || + path instanceof URL) { + return path.toString() + } +} + +function shouldAnalyze (path, fs) { + if (!path) return + + const notExcludedRootOp = !fs.opExcluded && fs.root + return notExcludedRootOp && (isAbsolute(path) || path.includes('../') || shouldAnalyzeURLFile(path, fs)) +} + +function shouldAnalyzeURLFile (path, fs) { + if (path.startsWith('file://')) { + return shouldAnalyze(path.substring(7), fs) + } +} + +module.exports = { + enable, + disable +} diff --git a/packages/dd-trace/src/appsec/rasp/utils.js b/packages/dd-trace/src/appsec/rasp/utils.js index 2a46b76d6e4..c4ee4f55c3f 100644 --- a/packages/dd-trace/src/appsec/rasp/utils.js +++ b/packages/dd-trace/src/appsec/rasp/utils.js @@ -13,7 +13,8 @@ if (abortOnUncaughtException) { const RULE_TYPES = { SSRF: 'ssrf', - SQL_INJECTION: 'sql_injection' + SQL_INJECTION: 'sql_injection', + LFI: 'lfi' } class DatadogRaspAbortError extends Error { diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index f42d7358203..05dc96233fd 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -19,5 +19,6 @@ module.exports = { APM_TRACING_ENABLED: 1n << 19n, ASM_RASP_SQLI: 1n << 21n, ASM_RASP_SSRF: 1n << 23n, + ASM_RASP_LFI: 1n << 24n, APM_TRACING_SAMPLE_RULES: 1n << 29n } diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index b63b3690102..28772c60c2e 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -79,6 +79,7 @@ function enableWafUpdate (appsecConfig) { if (appsecConfig.rasp?.enabled) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, true) } // TODO: delete noop handlers and kPreUpdate and replace with batched handlers @@ -106,6 +107,7 @@ function disableWafUpdate () { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_LFI, false) rc.removeProductHandler('ASM_DATA') rc.removeProductHandler('ASM_DD') diff --git a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js index 5b46c193fbd..6c39799f916 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/path-traversal-analyzer.spec.js @@ -45,6 +45,14 @@ const InjectionAnalyzer = proxyquire('../../../../src/appsec/iast/analyzers/inje }) describe('path-traversal-analyzer', () => { + before(() => { + pathTraversalAnalyzer.enable() + }) + + after(() => { + pathTraversalAnalyzer.disable() + }) + it('Analyzer should be subscribed to proper channel', () => { expect(pathTraversalAnalyzer._subscriptions).to.have.lengthOf(1) expect(pathTraversalAnalyzer._subscriptions[0]._channel.name).to.equals('apm:fs:operation:start') diff --git a/packages/dd-trace/test/appsec/iast/index.spec.js b/packages/dd-trace/test/appsec/iast/index.spec.js index 7035296d8de..f770694ede4 100644 --- a/packages/dd-trace/test/appsec/iast/index.spec.js +++ b/packages/dd-trace/test/appsec/iast/index.spec.js @@ -7,6 +7,7 @@ const iastContextFunctions = require('../../../src/appsec/iast/iast-context') const overheadController = require('../../../src/appsec/iast/overhead-controller') const vulnerabilityReporter = require('../../../src/appsec/iast/vulnerability-reporter') const { testInRequest } = require('./utils') +const { IAST_MODULE } = require('../../../src/appsec/rasp/fs-plugin') describe('IAST Index', () => { beforeEach(() => { @@ -102,6 +103,8 @@ describe('IAST Index', () => { let mockVulnerabilityReporter let mockIast let mockOverheadController + let appsecFsPlugin + let analyzers const config = new Config({ experimental: { @@ -125,9 +128,18 @@ describe('IAST Index', () => { startGlobalContext: sinon.stub(), finishGlobalContext: sinon.stub() } + appsecFsPlugin = { + enable: sinon.stub(), + disable: sinon.stub() + } + analyzers = { + enableAllAnalyzers: sinon.stub() + } mockIast = proxyquire('../../../src/appsec/iast', { './vulnerability-reporter': mockVulnerabilityReporter, - './overhead-controller': mockOverheadController + './overhead-controller': mockOverheadController, + '../rasp/fs-plugin': appsecFsPlugin, + './analyzers': analyzers }) }) @@ -136,6 +148,22 @@ describe('IAST Index', () => { mockIast.disable() }) + describe('enable', () => { + it('should enable AppsecFsPlugin', () => { + mockIast.enable(config) + expect(appsecFsPlugin.enable).to.have.been.calledOnceWithExactly(IAST_MODULE) + expect(analyzers.enableAllAnalyzers).to.have.been.calledAfter(appsecFsPlugin.enable) + }) + }) + + describe('disable', () => { + it('should disable AppsecFsPlugin', () => { + mockIast.enable(config) + mockIast.disable() + expect(appsecFsPlugin.disable).to.have.been.calledOnceWithExactly(IAST_MODULE) + }) + }) + describe('managing overhead controller global context', () => { it('should start global context refresher on iast enabled', () => { mockIast.enable(config) diff --git a/packages/dd-trace/test/appsec/index.spec.js b/packages/dd-trace/test/appsec/index.spec.js index b8a41d840b5..8548804ac38 100644 --- a/packages/dd-trace/test/appsec/index.spec.js +++ b/packages/dd-trace/test/appsec/index.spec.js @@ -1058,7 +1058,10 @@ describe('IP blocking', function () { beforeEach(() => { appsec.enable(new Config({ appsec: { - enabled: true + enabled: true, + rasp: { + enabled: false // disable rasp to not trigger lfi + } } })) diff --git a/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js b/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js new file mode 100644 index 00000000000..03b2a0acdd0 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/fs-plugin.spec.js @@ -0,0 +1,251 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { assert } = require('chai') +const path = require('path') +const dc = require('dc-polyfill') +const { storage } = require('../../../../datadog-core') +const { AppsecFsPlugin } = require('../../../src/appsec/rasp/fs-plugin') +const agent = require('../../plugins/agent') + +const opStartCh = dc.channel('apm:fs:operation:start') +const opFinishCh = dc.channel('apm:fs:operation:finish') + +describe('AppsecFsPlugin', () => { + let appsecFsPlugin + + beforeEach(() => { + appsecFsPlugin = new AppsecFsPlugin() + appsecFsPlugin.enable() + }) + + afterEach(() => { appsecFsPlugin.disable() }) + + describe('enable/disable', () => { + let fsPlugin, configure + + beforeEach(() => { + configure = sinon.stub() + class PluginClass { + addSub (channelName, handler) {} + + configure (config) { + configure(config) + } + } + + fsPlugin = proxyquire('../../../src/appsec/rasp/fs-plugin', { + '../../plugins/plugin': PluginClass + }) + }) + + afterEach(() => { sinon.restore() }) + + it('should require valid mod when calling enable', () => { + fsPlugin.enable('iast') + + sinon.assert.calledOnceWithExactly(configure, true) + }) + + it('should create only one instance', () => { + fsPlugin.enable('iast') + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + sinon.assert.calledOnceWithExactly(configure, true) + }) + + it('should discard unknown mods when enabled', () => { + fsPlugin.enable('unknown') + sinon.assert.notCalled(configure) + + fsPlugin.enable() + sinon.assert.notCalled(configure) + }) + + it('should not disable if there are still modules using the plugin', () => { + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + fsPlugin.disable('rasp') + + sinon.assert.calledOnce(configure) + }) + + it('should disable only if there are no more modules using the plugin', () => { + fsPlugin.enable('iast') + fsPlugin.enable('rasp') + + fsPlugin.disable('rasp') + fsPlugin.disable('iast') + + sinon.assert.calledTwice(configure) + assert.strictEqual(configure.secondCall.args[0], false) + }) + + it('should discard unknown mods when disabling', () => { + fsPlugin.disable('unknown') + sinon.assert.notCalled(configure) + + fsPlugin.disable() + sinon.assert.notCalled(configure) + }) + }) + + describe('_onFsOperationStart', () => { + it('should mark fs root', () => { + const origStore = {} + storage.enterWith(origStore) + + appsecFsPlugin._onFsOperationStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', origStore) + assert.propertyVal(store.fs, 'root', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, origStore) + assert.notProperty(store, 'fs') + }) + + it('should mark fs children', () => { + const origStore = { orig: true } + storage.enterWith(origStore) + + appsecFsPlugin._onFsOperationStart() + + const rootStore = storage.getStore() + assert.property(rootStore, 'fs') + assert.propertyVal(rootStore.fs, 'parentStore', origStore) + assert.propertyVal(rootStore.fs, 'root', true) + + appsecFsPlugin._onFsOperationStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', rootStore) + assert.propertyVal(store.fs, 'root', false) + assert.propertyVal(store, 'orig', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, rootStore) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + store = storage.getStore() + assert.equal(store, origStore) + }) + }) + + describe('_onResponseRenderStart', () => { + it('should mark fs ops as excluded while response rendering', () => { + appsecFsPlugin.enable() + + const origStore = {} + storage.enterWith(origStore) + + appsecFsPlugin._onResponseRenderStart() + + let store = storage.getStore() + assert.property(store, 'fs') + assert.propertyVal(store.fs, 'parentStore', origStore) + assert.propertyVal(store.fs, 'opExcluded', true) + + appsecFsPlugin._onFsOperationFinishOrRenderEnd() + + store = storage.getStore() + assert.equal(store, origStore) + assert.notProperty(store, 'fs') + }) + }) + + describe('integration', () => { + describe('apm:fs:operation', () => { + let fs + + afterEach(() => agent.close({ ritmReset: false })) + + beforeEach(() => agent.load('fs', undefined, { flushInterval: 1 }).then(() => { + fs = require('fs') + })) + + it('should mark root operations', () => { + let count = 0 + const onStart = () => { + const store = storage.getStore() + assert.isNotNull(store.fs) + + count++ + assert.strictEqual(count === 1, store.fs.root) + } + + try { + const origStore = {} + storage.enterWith(origStore) + + opStartCh.subscribe(onStart) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 4) + } finally { + opStartCh.unsubscribe(onStart) + } + }) + + it('should mark root even if op is excluded', () => { + let count = 0 + const onStart = () => { + const store = storage.getStore() + assert.isNotNull(store.fs) + + count++ + assert.isUndefined(store.fs.root) + } + + try { + const origStore = { + fs: { opExcluded: true } + } + storage.enterWith(origStore) + + opStartCh.subscribe(onStart) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 4) + } finally { + opStartCh.unsubscribe(onStart) + } + }) + + it('should clean up store when finishing op', () => { + let count = 4 + const onFinish = () => { + const store = storage.getStore() + count-- + + if (count === 0) { + assert.isUndefined(store.fs) + } + } + try { + const origStore = {} + storage.enterWith(origStore) + + opFinishCh.subscribe(onFinish) + + fs.readFileSync(path.join(__dirname, 'fs-plugin.spec.js')) + + assert.strictEqual(count, 0) + } finally { + opFinishCh.unsubscribe(onFinish) + } + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/index.spec.js b/packages/dd-trace/test/appsec/rasp/index.spec.js index 0dae9c527e5..be6c602780a 100644 --- a/packages/dd-trace/test/appsec/rasp/index.spec.js +++ b/packages/dd-trace/test/appsec/rasp/index.spec.js @@ -1,9 +1,12 @@ 'use strict' -const rasp = require('../../../src/appsec/rasp') +const proxyquire = require('proxyquire') const { handleUncaughtExceptionMonitor } = require('../../../src/appsec/rasp') +const { DatadogRaspAbortError } = require('../../../src/appsec/rasp/utils') describe('RASP', () => { + let rasp, subscribe, unsubscribe, block, blocked + beforeEach(() => { const config = { appsec: { @@ -15,6 +18,25 @@ describe('RASP', () => { } } + subscribe = sinon.stub() + unsubscribe = sinon.stub() + + block = sinon.stub() + + rasp = proxyquire('../../../src/appsec/rasp', { + '../blocking': { + block, + isBlocked: sinon.stub().callsFake(() => blocked) + }, + '../channels': { + expressMiddlewareError: { + subscribe, + unsubscribe, + hasSubscribers: true + } + } + }) + rasp.enable(config) }) @@ -31,4 +53,53 @@ describe('RASP', () => { handleUncaughtExceptionMonitor(err) }) }) + + describe('enable/disable', () => { + it('should subscribe to apm:express:middleware:error', () => { + sinon.assert.calledOnce(subscribe) + }) + + it('should unsubscribe to apm:express:middleware:error', () => { + rasp.disable() + + sinon.assert.calledOnce(unsubscribe) + }) + }) + + describe('blockOnDatadogRaspAbortError', () => { + let req, res, blockingAction + + beforeEach(() => { + req = {} + res = {} + blockingAction = {} + }) + + afterEach(() => { + sinon.restore() + }) + + it('should skip non DatadogRaspAbortError', () => { + rasp.blockOnDatadogRaspAbortError({ error: new Error() }) + + sinon.assert.notCalled(block) + }) + + it('should block DatadogRaspAbortError first time', () => { + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + sinon.assert.calledOnce(block) + }) + + it('should skip calling block if blocked before', () => { + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + blocked = true + + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + rasp.blockOnDatadogRaspAbortError({ error: new DatadogRaspAbortError(req, res, blockingAction) }) + + sinon.assert.calledOnce(block) + }) + }) }) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js new file mode 100644 index 00000000000..b5b825cc628 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.express.plugin.spec.js @@ -0,0 +1,469 @@ +'use strict' + +const Axios = require('axios') +const os = require('os') +const fs = require('fs') +const agent = require('../../plugins/agent') +const appsec = require('../../../src/appsec') +const Config = require('../../../src/config') +const path = require('path') +const { assert } = require('chai') +const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require('./utils') + +describe('RASP - lfi', () => { + let axios + + async function testBlockingRequest (url = '/?file=/test.file', config = undefined, ruleEvalCount = 1) { + try { + await axios.get(url, config) + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 418) // a teapot + + return checkRaspExecutedAndHasThreat(agent, 'rasp-lfi-rule-id-1', ruleEvalCount) + } + + assert.fail('Request should be blocked') + } + + withVersions('express', 'express', expressVersion => { + let app, server + + before(() => { + return agent.load(['http', 'express'], { client: false }) + }) + + before((done) => { + const express = require(`../../../../../versions/express@${expressVersion}`).get() + const expressApp = express() + + expressApp.get('/', (req, res) => { + app(req, res) + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'lfi_rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server = expressApp.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + describe('lfi', () => { + function getApp (fn, args, options) { + return async (req, res) => { + try { + const result = await fn(args) + options.onfinish?.(result) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(418) + } + } + res.end('end') + } + } + + function getAppSync (fn, args, options) { + return (req, res) => { + try { + const result = fn(args) + options.onfinish?.(result) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(418) + } + } + res.end('end') + } + } + + function runFsMethodTest (description, options, fn, ...args) { + const { vulnerableIndex = 0, ruleEvalCount } = options + + describe(description, () => { + const getAppFn = options.getAppFn ?? getApp + + it('should block param from the request', async () => { + app = getAppFn(fn, args, options) + + const file = args[vulnerableIndex] + return testBlockingRequest(`/?file=${file}`, undefined, ruleEvalCount) + .then(span => { + assert(span.meta['_dd.appsec.json'].includes(file)) + }) + }) + + it('should not block if param not found in the request', async () => { + app = getAppFn(fn, args, options) + + await axios.get('/?file=/test.file') + + return checkRaspExecutedAndNotThreat(agent, false) + }) + }) + } + + function runFsMethodTestThreeWay (methodName, options = {}, ...args) { + let desc = `test ${methodName} ${options.desc ?? ''}` + const { vulnerableIndex = 0 } = options + if (vulnerableIndex !== 0) { + desc += ` with vulnerable index ${vulnerableIndex}` + } + describe(desc, () => { + runFsMethodTest(`test fs.${methodName}Sync method`, { ...options, getAppFn: getAppSync }, (args) => { + return require('fs')[`${methodName}Sync`](...args) + }, ...args) + + runFsMethodTest(`test fs.${methodName} method`, options, (args) => { + return new Promise((resolve, reject) => { + require('fs')[methodName](...args, (err, res) => { + if (err) reject(err) + else resolve(res) + }) + }) + }, ...args) + + runFsMethodTest(`test fs.promises.${methodName} method`, options, async (args) => { + return require('fs').promises[methodName](...args) + }, ...args) + }) + } + + function unlink (...args) { + args.forEach(arg => { + try { + fs.unlinkSync(arg) + } catch (e) { + + } + }) + } + + describe('test access', () => { + runFsMethodTestThreeWay('access', undefined, __filename) + runFsMethodTestThreeWay('access', { desc: 'Buffer' }, Buffer.from(__filename)) + + // not supported by waf yet + // runFsMethodTestThreeWay('access', { desc: 'URL' }, new URL(`file://${__filename}`)) + }) + + describe('test appendFile', () => { + const filename = path.join(os.tmpdir(), 'test-appendfile') + + beforeEach(() => { + fs.writeFileSync(filename, '') + }) + + afterEach(() => { + fs.unlinkSync(filename) + }) + + runFsMethodTestThreeWay('appendFile', undefined, filename, 'test-content') + }) + + describe('test chmod', () => { + const filename = path.join(os.tmpdir(), 'test-chmod') + + beforeEach(() => { + fs.writeFileSync(filename, '') + }) + + afterEach(() => { + fs.unlinkSync(filename) + }) + runFsMethodTestThreeWay('chmod', undefined, filename, '666') + }) + + describe('test copyFile', () => { + const src = path.join(os.tmpdir(), 'test-copyFile-src') + const dest = path.join(os.tmpdir(), 'test-copyFile-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test link', () => { + const src = path.join(os.tmpdir(), 'test-link-src') + const dest = path.join(os.tmpdir(), 'test-link-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('copyFile', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test lstat', () => { + runFsMethodTestThreeWay('lstat', undefined, __filename) + }) + + describe('test mkdir', () => { + const dirname = path.join(os.tmpdir(), 'test-mkdir') + + afterEach(() => { + try { + fs.rmdirSync(dirname) + } catch (e) { + // some ops are blocked + } + }) + runFsMethodTestThreeWay('mkdir', undefined, dirname) + }) + + describe('test mkdtemp', () => { + const dirname = path.join(os.tmpdir(), 'test-mkdtemp') + + runFsMethodTestThreeWay('mkdtemp', { + onfinish: (todelete) => { + try { + fs.rmdirSync(todelete) + } catch (e) { + // some ops are blocked + } + } + }, dirname) + }) + + describe('test open', () => { + runFsMethodTestThreeWay('open', { + onfinish: (fd) => { + if (fd && fd.close) { + fd.close() + } else { + fs.close(fd, () => {}) + } + } + }, __filename, 'r') + }) + + describe('test opendir', () => { + const dirname = path.join(os.tmpdir(), 'test-opendir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + fs.rmdirSync(dirname) + }) + runFsMethodTestThreeWay('opendir', { + onfinish: (dir) => { + dir.close() + } + }, dirname) + }) + + describe('test readdir', () => { + const dirname = path.join(os.tmpdir(), 'test-opendir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + fs.rmdirSync(dirname) + }) + runFsMethodTestThreeWay('readdir', undefined, dirname) + }) + + describe('test readFile', () => { + runFsMethodTestThreeWay('readFile', undefined, __filename) + }) + + describe('test readlink', () => { + const src = path.join(os.tmpdir(), 'test-readlink-src') + const dest = path.join(os.tmpdir(), 'test-readlink-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + fs.linkSync(src, dest) + }) + + afterEach(() => unlink(src, dest)) + + runFsMethodTestThreeWay('readlink', undefined, dest) + }) + + describe('test realpath', () => { + runFsMethodTestThreeWay('realpath', undefined, __filename) + + runFsMethodTest('test fs.realpath.native method', {}, (args) => { + return new Promise((resolve, reject) => { + require('fs').realpath.native(...args, (err, result) => { + if (err) reject(err) + else resolve(result) + }) + }) + }, __filename) + }) + + describe('test rename', () => { + const src = path.join(os.tmpdir(), 'test-rename-src') + const dest = path.join(os.tmpdir(), 'test-rename-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => unlink(dest)) + + runFsMethodTestThreeWay('rename', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('rename', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test rmdir', () => { + const dirname = path.join(os.tmpdir(), 'test-rmdir') + + beforeEach(() => { + fs.mkdirSync(dirname) + }) + + afterEach(() => { + try { fs.rmdirSync(dirname) } catch (e) {} + }) + + runFsMethodTestThreeWay('rmdir', undefined, dirname) + }) + + describe('test stat', () => { + runFsMethodTestThreeWay('stat', undefined, __filename) + }) + + describe('test symlink', () => { + const src = path.join(os.tmpdir(), 'test-symlink-src') + const dest = path.join(os.tmpdir(), 'test-symlink-dst') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + + afterEach(() => { + unlink(src, dest) + }) + + runFsMethodTestThreeWay('symlink', { vulnerableIndex: 0, ruleEvalCount: 2 }, src, dest) + runFsMethodTestThreeWay('symlink', { vulnerableIndex: 1, ruleEvalCount: 2 }, src, dest) + }) + + describe('test truncate', () => { + const src = path.join(os.tmpdir(), 'test-truncate-src') + + beforeEach(() => { + fs.writeFileSync(src, 'aaaaaa') + }) + + afterEach(() => unlink(src)) + + runFsMethodTestThreeWay('truncate', undefined, src) + }) + + describe('test unlink', () => { + const src = path.join(os.tmpdir(), 'test-unlink-src') + + beforeEach(() => { + fs.writeFileSync(src, '') + }) + runFsMethodTestThreeWay('unlink', undefined, src) + }) + + describe('test writeFile', () => { + const src = path.join(os.tmpdir(), 'test-writeFile-src') + + afterEach(() => unlink(src)) + + runFsMethodTestThreeWay('writeFile', undefined, src, 'content') + }) + }) + }) + + describe('without express', () => { + let app, server + + before(() => { + return agent.load(['http'], { client: false }) + }) + + before((done) => { + const http = require('http') + server = http.createServer((req, res) => { + if (app) { + app(req, res) + } else { + res.end('end') + } + }) + + appsec.enable(new Config({ + appsec: { + enabled: true, + rules: path.join(__dirname, 'resources', 'lfi_rasp_rules.json'), + rasp: { enabled: true } + } + })) + + server.listen(0, () => { + const port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + + done() + }) + }) + + after(() => { + appsec.disable() + server.close() + return agent.close({ ritmReset: false }) + }) + + it('Should detect threat but not block', async () => { + app = (req, res) => { + try { + require('fs').statSync(req.headers.file) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + res.writeHead(500) + } else { + res.writeHead(418) + } + } + res.end('end') + } + + return testBlockingRequest('/', { + headers: { + file: '/test.file' + } + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js new file mode 100644 index 00000000000..45dc1cac46f --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.integration.express.plugin.spec.js @@ -0,0 +1,69 @@ +'use strict' + +const { createSandbox, FakeAgent, spawnProc } = require('../../../../../integration-tests/helpers') +const getPort = require('get-port') +const path = require('path') +const Axios = require('axios') +const { assert } = require('chai') + +describe('RASP - lfi - integration - sync', () => { + let axios, sandbox, cwd, appPort, appFile, agent, proc + + before(async function () { + this.timeout(60000) + sandbox = await createSandbox( + ['express', 'fs'], + false, + [path.join(__dirname, 'resources')]) + + appPort = await getPort() + cwd = sandbox.folder + appFile = path.join(cwd, 'resources', 'lfi-app', 'index.js') + + axios = Axios.create({ + baseURL: `http://localhost:${appPort}` + }) + }) + + after(async function () { + this.timeout(60000) + await sandbox.remove() + }) + + beforeEach(async () => { + agent = await new FakeAgent().start() + proc = await spawnProc(appFile, { + cwd, + env: { + DD_TRACE_AGENT_PORT: agent.port, + APP_PORT: appPort, + DD_APPSEC_ENABLED: true, + DD_APPSEC_RASP_ENABLED: true, + DD_APPSEC_RULES: path.join(cwd, 'resources', 'lfi_rasp_rules.json') + } + }) + }) + + afterEach(async () => { + proc.kill() + await agent.stop() + }) + + it('should block a sync endpoint getting the error from apm:express:middleware:error', async () => { + try { + await axios.get('/lfi/sync?file=/etc/passwd') + } catch (e) { + if (!e.response) { + throw e + } + + assert.strictEqual(e.response.status, 403) + return await agent.assertMessageReceived(({ headers, payload }) => { + assert.property(payload[0][0].meta, '_dd.appsec.json') + assert.include(payload[0][0].meta['_dd.appsec.json'], '"rasp-lfi-rule-id-1"') + }) + } + + throw new Error('Request should be blocked') + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/lfi.spec.js b/packages/dd-trace/test/appsec/rasp/lfi.spec.js new file mode 100644 index 00000000000..405311ae0d3 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/lfi.spec.js @@ -0,0 +1,144 @@ +'use strict' + +const proxyquire = require('proxyquire') +const { assert } = require('chai') +const { fsOperationStart, incomingHttpRequestStart } = require('../../../src/appsec/channels') +const { FS_OPERATION_PATH } = require('../../../src/appsec/addresses') +const { RASP_MODULE } = require('../../../src/appsec/rasp/fs-plugin') + +describe('RASP - lfi.js', () => { + let waf, datadogCore, lfi, web, blocking, appsecFsPlugin, config + + beforeEach(() => { + datadogCore = { + storage: { + getStore: sinon.stub() + } + } + + waf = { + run: sinon.stub() + } + + web = { + root: sinon.stub() + } + + blocking = { + block: sinon.stub() + } + + appsecFsPlugin = { + enable: sinon.stub(), + disable: sinon.stub() + } + + lfi = proxyquire('../../../src/appsec/rasp/lfi', { + '../../../../datadog-core': datadogCore, + '../waf': waf, + '../../plugins/util/web': web, + '../blocking': blocking, + './fs-plugin': appsecFsPlugin + }) + + config = { + appsec: { + stackTrace: { + enabled: true, + maxStackTraces: 2, + maxDepth: 42 + } + } + } + }) + + afterEach(() => { + sinon.restore() + lfi.disable() + }) + + describe('enable', () => { + it('should subscribe to first http req', () => { + const subscribe = sinon.stub(incomingHttpRequestStart, 'subscribe') + + lfi.enable(config) + + sinon.assert.calledOnce(subscribe) + }) + + it('should enable AppsecFsPlugin after the first request', () => { + const unsubscribe = sinon.stub(incomingHttpRequestStart, 'unsubscribe') + const fsOpSubscribe = sinon.stub(fsOperationStart, 'subscribe') + + lfi.enable(config) + + incomingHttpRequestStart.publish({}) + + sinon.assert.calledOnceWithExactly(appsecFsPlugin.enable, RASP_MODULE) + + assert(fsOpSubscribe.calledAfter(appsecFsPlugin.enable)) + + process.nextTick(() => { + sinon.assert.calledOnce(unsubscribe) + }) + }) + }) + + describe('disable', () => { + it('should disable AppsecFsPlugin', () => { + lfi.enable(config) + + lfi.disable() + sinon.assert.calledOnceWithExactly(appsecFsPlugin.disable, RASP_MODULE) + }) + }) + + describe('analyzeLfi', () => { + const path = '/etc/passwd' + const ctx = { path } + const req = {} + + beforeEach(() => { + lfi.enable(config) + + incomingHttpRequestStart.publish({}) + }) + + it('should analyze lfi for root fs operations', () => { + const fs = { root: true } + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + const persistent = { [FS_OPERATION_PATH]: path } + sinon.assert.calledOnceWithExactly(waf.run, { persistent }, req, 'lfi') + }) + + it('should NOT analyze lfi for child fs operations', () => { + const fs = {} + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should NOT analyze lfi for undefined fs (AppsecFsPlugin disabled)', () => { + const fs = undefined + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + + it('should NOT analyze lfi for excluded operations', () => { + const fs = { opExcluded: true, root: true } + datadogCore.storage.getStore.returns({ req, fs }) + + fsOperationStart.publish(ctx) + + sinon.assert.notCalled(waf.run) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js b/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js new file mode 100644 index 00000000000..1beb4d977cb --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/lfi-app/index.js @@ -0,0 +1,28 @@ +'use strict' + +const tracer = require('dd-trace') +tracer.init({ + flushInterval: 0 +}) + +const express = require('express') +const { readFileSync } = require('fs') + +const app = express() +const port = process.env.APP_PORT || 3000 + +app.get('/lfi/sync', (req, res) => { + let result + try { + result = readFileSync(req.query.file) + } catch (e) { + if (e.message === 'DatadogRaspAbortError') { + throw e + } + } + res.send(result) +}) + +app.listen(port, () => { + process.send({ port }) +}) diff --git a/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json b/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json new file mode 100644 index 00000000000..814f6c72236 --- /dev/null +++ b/packages/dd-trace/test/appsec/rasp/resources/lfi_rasp_rules.json @@ -0,0 +1,61 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.99.0" + }, + "rules": [ + { + "id": "rasp-lfi-rule-id-1", + "name": "Local file inclusion exploit", + "enabled": true, + "tags": { + "type": "lfi", + "category": "vulnerability_trigger", + "cwe": "22", + "capec": "1000/255/153/126", + "confidence": "0", + "module": "rasp" + }, + "conditions": [ + { + "parameters": { + "resource": [ + { + "address": "server.io.fs.file" + } + ], + "params": [ + { + "address": "server.request.query" + }, + { + "address": "server.request.body" + }, + { + "address": "server.request.path_params" + }, + { + "address": "grpc.server.request.message" + }, + { + "address": "server.request.headers.no_cookies" + }, + { + "address": "graphql.server.all_resolvers" + }, + { + "address": "graphql.server.resolver" + } + ] + }, + "operator": "lfi_detector" + } + ], + "transformers": [], + "on_match": [ + "block", + "stack_trace" + ] + } + ] +} diff --git a/packages/dd-trace/test/appsec/rasp/utils.js b/packages/dd-trace/test/appsec/rasp/utils.js index e9353d5d815..0d8a3e076a4 100644 --- a/packages/dd-trace/test/appsec/rasp/utils.js +++ b/packages/dd-trace/test/appsec/rasp/utils.js @@ -13,24 +13,28 @@ function getWebSpan (traces) { throw new Error('web span not found') } -function checkRaspExecutedAndNotThreat (agent) { +function checkRaspExecutedAndNotThreat (agent, checkRuleEval = true) { return agent.use((traces) => { const span = getWebSpan(traces) assert.notProperty(span.meta, '_dd.appsec.json') assert.notProperty(span.meta_struct || {}, '_dd.stack') - assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + if (checkRuleEval) { + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + } }) } -function checkRaspExecutedAndHasThreat (agent, ruleId) { +function checkRaspExecutedAndHasThreat (agent, ruleId, ruleEvalCount = 1) { return agent.use((traces) => { const span = getWebSpan(traces) assert.property(span.meta, '_dd.appsec.json') assert(span.meta['_dd.appsec.json'].includes(ruleId)) - assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], 1) + assert.equal(span.metrics['_dd.appsec.rasp.rule.eval'], ruleEvalCount) assert(span.metrics['_dd.appsec.rasp.duration'] > 0) assert(span.metrics['_dd.appsec.rasp.duration_ext'] > 0) assert.property(span.meta_struct, '_dd.stack') + + return span }) } diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index fd923c9a92b..c3da43a17c0 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -290,6 +290,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') @@ -324,6 +326,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) expect(rc.setProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.setProductHandler).to.have.been.calledWith('ASM_DD') @@ -360,6 +364,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, true) }) it('should not activate rasp capabilities if rasp is disabled', () => { @@ -391,6 +397,8 @@ describe('Remote Config index', () => { .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_RASP_SSRF) expect(rc.updateCapabilities) .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_RASP_SQLI) + expect(rc.updateCapabilities) + .to.not.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI) }) }) @@ -422,6 +430,8 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, false) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SQLI, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_LFI, false) expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DATA') expect(rc.removeProductHandler).to.have.been.calledWith('ASM_DD') diff --git a/packages/dd-trace/test/appsec/response_blocking.spec.js b/packages/dd-trace/test/appsec/response_blocking.spec.js index 2868a42b05b..03541858955 100644 --- a/packages/dd-trace/test/appsec/response_blocking.spec.js +++ b/packages/dd-trace/test/appsec/response_blocking.spec.js @@ -52,7 +52,10 @@ describe('HTTP Response Blocking', () => { appsec.enable(new Config({ appsec: { enabled: true, - rules: path.join(__dirname, 'response_blocking_rules.json') + rules: path.join(__dirname, 'response_blocking_rules.json'), + rasp: { + enabled: false // disable rasp to not trigger waf.run executions due to lfi + } } })) }) diff --git a/packages/dd-trace/test/plugins/agent.js b/packages/dd-trace/test/plugins/agent.js index dc87f18dc3a..cb6f241e7d3 100644 --- a/packages/dd-trace/test/plugins/agent.js +++ b/packages/dd-trace/test/plugins/agent.js @@ -210,10 +210,10 @@ function runCallback (callback, options, handlers) { function handler () { try { - callback.apply(null, arguments) + const result = callback.apply(null, arguments) handlers.delete(handlerPayload) clearTimeout(timeout) - deferred.resolve() + deferred.resolve(result) } catch (e) { if (options && options.rejectFirst) { clearTimeout(timeout) From a2b318df2713264e8742deca3fab4d447657514d Mon Sep 17 00:00:00 2001 From: Carles Capell <107924659+CarlesDD@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:06:18 +0200 Subject: [PATCH 23/27] [ASM] Add support for attacker fingerprinting (#4698) * Report WAF fingerprints * WAF fingerprint RC capabilities * Linting * Remove useless file * Add blank line * Remove unused capability * Generate fingerprint on user login events * Fix linting * Add passport plugin test to GHA * Add business logic addressses * Add body-parser dep to passport plugin test * Reformat test * Refactor report derivatives * Move method to its right place * Unify reportSchemas and reportFingerprint test in one suite * Unify reportSchemas and reportFingerprint test in one suite --- .github/workflows/appsec.yml | 14 ++ packages/dd-trace/src/appsec/addresses.js | 5 +- .../src/appsec/remote_config/capabilities.js | 5 +- .../src/appsec/remote_config/index.js | 6 + packages/dd-trace/src/appsec/reporter.js | 17 +- .../dd-trace/src/appsec/sdk/track_event.js | 5 + .../src/appsec/waf/waf_context_wrapper.js | 2 +- .../appsec/attacker-fingerprinting-rules.json | 204 ++++++++++++++++++ ...cker-fingerprinting.express.plugin.spec.js | 79 +++++++ ...ingerprinting.passport-http.plugin.spec.js | 107 +++++++++ ...ngerprinting.passport-local.plugin.spec.js | 105 +++++++++ .../appsec/attacker-fingerprinting.spec.js | 83 +++++++ .../test/appsec/remote_config/index.spec.js | 30 +++ .../dd-trace/test/appsec/reporter.spec.js | 16 +- .../test/appsec/sdk/track_event.spec.js | 31 ++- .../dd-trace/test/appsec/waf/index.spec.js | 26 ++- packages/dd-trace/test/plugins/externals.json | 4 + 17 files changed, 724 insertions(+), 15 deletions(-) create mode 100644 packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json create mode 100644 packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 19470023010..f41b18f9d53 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -250,3 +250,17 @@ jobs: - run: yarn test:integration:appsec - uses: ./.github/actions/node/latest - run: yarn test:integration:appsec + + passport: + runs-on: ubuntu-latest + env: + PLUGINS: passport-local|passport-http + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/node/setup + - uses: ./.github/actions/install + - uses: ./.github/actions/node/oldest + - run: yarn test:appsec:plugins:ci + - uses: ./.github/actions/node/latest + - run: yarn test:appsec:plugins:ci + - uses: codecov/codecov-action@v3 diff --git a/packages/dd-trace/src/appsec/addresses.js b/packages/dd-trace/src/appsec/addresses.js index f8ce3033d36..40c643012ef 100644 --- a/packages/dd-trace/src/appsec/addresses.js +++ b/packages/dd-trace/src/appsec/addresses.js @@ -26,5 +26,8 @@ module.exports = { FS_OPERATION_PATH: 'server.io.fs.file', DB_STATEMENT: 'server.db.statement', - DB_SYSTEM: 'server.db.system' + DB_SYSTEM: 'server.db.system', + + LOGIN_SUCCESS: 'server.business_logic.users.login.success', + LOGIN_FAILURE: 'server.business_logic.users.login.failure' } diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index 05dc96233fd..97965fb1203 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -20,5 +20,8 @@ module.exports = { ASM_RASP_SQLI: 1n << 21n, ASM_RASP_SSRF: 1n << 23n, ASM_RASP_LFI: 1n << 24n, - APM_TRACING_SAMPLE_RULES: 1n << 29n + APM_TRACING_SAMPLE_RULES: 1n << 29n, + ASM_ENDPOINT_FINGERPRINT: 1n << 32n, + ASM_NETWORK_FINGERPRINT: 1n << 34n, + ASM_HEADER_FINGERPRINT: 1n << 35n } diff --git a/packages/dd-trace/src/appsec/remote_config/index.js b/packages/dd-trace/src/appsec/remote_config/index.js index 28772c60c2e..2b7eea57c82 100644 --- a/packages/dd-trace/src/appsec/remote_config/index.js +++ b/packages/dd-trace/src/appsec/remote_config/index.js @@ -75,6 +75,9 @@ function enableWafUpdate (appsecConfig) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) if (appsecConfig.rasp?.enabled) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true) @@ -104,6 +107,9 @@ function disableWafUpdate () { rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_RULES, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false) diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index a58335d9ba7..dd2bde9fb06 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -153,7 +153,11 @@ function reportAttack (attackData) { rootSpan.addTags(newTags) } -function reportSchemas (derivatives) { +function isFingerprintDerivative (derivative) { + return derivative.startsWith('_dd.appsec.fp') +} + +function reportDerivatives (derivatives) { if (!derivatives) return const req = storage.getStore()?.req @@ -162,9 +166,12 @@ function reportSchemas (derivatives) { if (!rootSpan) return const tags = {} - for (const [address, value] of Object.entries(derivatives)) { - const gzippedValue = zlib.gzipSync(JSON.stringify(value)) - tags[address] = gzippedValue.toString('base64') + for (let [tag, value] of Object.entries(derivatives)) { + if (!isFingerprintDerivative(tag)) { + const gzippedValue = zlib.gzipSync(JSON.stringify(value)) + value = gzippedValue.toString('base64') + } + tags[tag] = value } rootSpan.addTags(tags) @@ -248,7 +255,7 @@ module.exports = { reportMetrics, reportAttack, reportWafUpdate: incrementWafUpdatesMetric, - reportSchemas, + reportDerivatives, finishRequest, setRateLimit, mapHeaderAndTags diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 61500e2cfbe..36c40093b19 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -5,6 +5,7 @@ const { getRootSpan } = require('./utils') const { MANUAL_KEEP } = require('../../../../../ext/tags') const { setUserTags } = require('./set_user') const standalone = require('../standalone') +const waf = require('../waf') function trackUserLoginSuccessEvent (tracer, user, metadata) { // TODO: better user check here and in _setUser() ? @@ -76,6 +77,10 @@ function trackEvent (eventName, fields, sdkMethodName, rootSpan, mode) { rootSpan.addTags(tags) standalone.sample(rootSpan) + + if (['users.login.success', 'users.login.failure'].includes(eventName)) { + waf.run({ persistent: { [`server.business_logic.${eventName}`]: null } }) + } } module.exports = { diff --git a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js index ed946633174..a2dae737a86 100644 --- a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +++ b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js @@ -93,7 +93,7 @@ class WAFContextWrapper { Reporter.reportAttack(JSON.stringify(result.events)) } - Reporter.reportSchemas(result.derivatives) + Reporter.reportDerivatives(result.derivatives) if (wafRunFinished.hasSubscribers) { wafRunFinished.publish({ payload }) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json b/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json new file mode 100644 index 00000000000..722f9153ce4 --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting-rules.json @@ -0,0 +1,204 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "tst-000-001-", + "name": "rule to test fingerprint", + "tags": { + "type": "attack_tool", + "category": "attack_attempt", + "confidence": "1" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.query" + } + ], + "list": [ + "testattack" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": [] + } + ], + "processors": [ + { + "id": "http-endpoint-fingerprint", + "generator": "http_endpoint_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "method": [ + { + "address": "server.request.method" + } + ], + "uri_raw": [ + { + "address": "server.request.uri.raw" + } + ], + "body": [ + { + "address": "server.request.body" + } + ], + "query": [ + { + "address": "server.request.query" + } + ], + "output": "_dd.appsec.fp.http.endpoint" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-header-fingerprint", + "generator": "http_header_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.header" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "http-network-fingerprint", + "generator": "http_network_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "headers": [ + { + "address": "server.request.headers.no_cookies" + } + ], + "output": "_dd.appsec.fp.http.network" + } + ] + }, + "evaluate": false, + "output": true + }, + { + "id": "session-fingerprint", + "generator": "session_fingerprint", + "conditions": [ + { + "operator": "exists", + "parameters": { + "inputs": [ + { + "address": "waf.context.event" + }, + { + "address": "server.business_logic.users.login.failure" + }, + { + "address": "server.business_logic.users.login.success" + } + ] + } + } + ], + "parameters": { + "mappings": [ + { + "cookies": [ + { + "address": "server.request.cookies" + } + ], + "session_id": [ + { + "address": "usr.session_id" + } + ], + "user_id": [ + { + "address": "usr.id" + } + ], + "output": "_dd.appsec.fp.session" + } + ] + }, + "evaluate": false, + "output": true + } + ] +} diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js new file mode 100644 index 00000000000..bc7c918965c --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.express.plugin.spec.js @@ -0,0 +1,79 @@ +'use strict' + +const axios = require('axios') +const { assert } = require('chai') +const path = require('path') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +describe('Attacker fingerprinting', () => { + let port, server + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + + const app = express() + app.use(bodyParser.json()) + + app.post('/', (req, res) => { + res.end('DONE') + }) + + server = app.listen(port, () => { + port = server.address().port + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + beforeEach(() => { + appsec.enable(new Config( + { + appsec: { + enabled: true, + rules: path.join(__dirname, 'attacker-fingerprinting-rules.json') + } + } + )) + }) + + afterEach(() => { + appsec.disable() + }) + + it('should report http fingerprints', async () => { + await axios.post( + `http://localhost:${port}/?key=testattack`, + { + bodyParam: 'bodyValue' + }, + { + headers: { + headerName: 'headerValue', + 'x-real-ip': '255.255.255.255' + } + } + ) + + await agent.use((traces) => { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-5-55682ec1') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-1-0100000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-8a5edab2-2c70e12b-be31090f') + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js new file mode 100644 index 00000000000..58b54e2c704 --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-http.plugin.spec.js @@ -0,0 +1,107 @@ +'use strict' + +const Axios = require('axios') +const { assert } = require('chai') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +function assertFingerprintInTraces (traces) { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-5-e58aa9dd') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-7e93fba0--') +} + +withVersions('passport-http', 'passport-http', version => { + describe('Attacker fingerprinting', () => { + let port, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before(() => { + appsec.enable(new Config({ + appsec: true + })) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + const passport = require('../../../../versions/passport').get() + const { BasicStrategy } = require(`../../../../versions/passport-http@${version}`).get() + + const app = express() + app.use(bodyParser.json()) + app.use(passport.initialize()) + + passport.use(new BasicStrategy( + function verify (username, password, done) { + if (username === 'success') { + done(null, { + id: 1234, + username + }) + } else { + done(null, false) + } + } + )) + + app.post('/login', passport.authenticate('basic', { session: false }), function (req, res) { + res.end() + }) + + server = app.listen(port, () => { + port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + after(() => { + appsec.disable() + }) + + it('should report http fingerprints on login fail', async () => { + try { + await axios.post( + `http://localhost:${port}/login`, {}, { + auth: { + username: 'fail', + password: '1234' + } + } + ) + } catch (e) {} + + await agent.use(assertFingerprintInTraces) + }) + + it('should report http fingerprints on login successful', async () => { + await axios.post( + `http://localhost:${port}/login`, {}, { + auth: { + username: 'success', + password: '1234' + } + } + ) + + await agent.use(assertFingerprintInTraces) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js new file mode 100644 index 00000000000..b51aa57de9c --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.passport-local.plugin.spec.js @@ -0,0 +1,105 @@ +'use strict' + +const Axios = require('axios') +const { assert } = require('chai') + +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +function assertFingerprintInTraces (traces) { + const span = traces[0][0] + assert.property(span.meta, '_dd.appsec.fp.http.header') + assert.equal(span.meta['_dd.appsec.fp.http.header'], 'hdr-0110000110-6431a3e6-4-c348f529') + assert.property(span.meta, '_dd.appsec.fp.http.network') + assert.equal(span.meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + assert.property(span.meta, '_dd.appsec.fp.http.endpoint') + assert.equal(span.meta['_dd.appsec.fp.http.endpoint'], 'http-post-7e93fba0--f29f6224') +} + +withVersions('passport-local', 'passport-local', version => { + describe('Attacker fingerprinting', () => { + let port, server, axios + + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + before(() => { + appsec.enable(new Config({ + appsec: true + })) + }) + + before((done) => { + const express = require('../../../../versions/express').get() + const bodyParser = require('../../../../versions/body-parser').get() + const passport = require('../../../../versions/passport').get() + const LocalStrategy = require(`../../../../versions/passport-local@${version}`).get() + + const app = express() + app.use(bodyParser.json()) + app.use(passport.initialize()) + + passport.use(new LocalStrategy( + function verify (username, password, done) { + if (username === 'success') { + done(null, { + id: 1234, + username + }) + } else { + done(null, false) + } + } + )) + + app.post('/login', passport.authenticate('local', { session: false }), function (req, res) { + res.end() + }) + + server = app.listen(port, () => { + port = server.address().port + axios = Axios.create({ + baseURL: `http://localhost:${port}` + }) + done() + }) + }) + + after(() => { + server.close() + return agent.close({ ritmReset: false }) + }) + + after(() => { + appsec.disable() + }) + + it('should report http fingerprints on login fail', async () => { + try { + await axios.post( + `http://localhost:${port}/login`, + { + username: 'fail', + password: '1234' + } + ) + } catch (e) {} + + await agent.use(assertFingerprintInTraces) + }) + + it('should report http fingerprints on login successful', async () => { + await axios.post( + `http://localhost:${port}/login`, + { + username: 'success', + password: '1234' + } + ) + + await agent.use(assertFingerprintInTraces) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js b/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js new file mode 100644 index 00000000000..013c9cbd3ed --- /dev/null +++ b/packages/dd-trace/test/appsec/attacker-fingerprinting.spec.js @@ -0,0 +1,83 @@ +'use strict' + +const axios = require('axios') +const { assert } = require('chai') +const agent = require('../plugins/agent') +const tracer = require('../../../../index') +const appsec = require('../../src/appsec') +const Config = require('../../src/config') + +describe('Attacker fingerprinting', () => { + describe('SDK', () => { + let http + let controller + let appListener + let port + + function listener (req, res) { + if (controller) { + controller(req, res) + } + } + + before(() => { + appsec.enable(new Config({ + enabled: true + })) + }) + + before(async () => { + await agent.load('http') + http = require('http') + }) + + before(done => { + const server = new http.Server(listener) + appListener = server + .listen(port, 'localhost', () => { + port = appListener.address().port + done() + }) + }) + + after(() => { + appListener.close() + appsec.disable() + return agent.close({ ritmReset: false }) + }) + + it('should provide fingerprinting on successful user login track', (done) => { + controller = (req, res) => { + tracer.appsec.trackUserLoginSuccessEvent({ + id: 'test_user_id' + }, { metakey: 'metaValue' }) + res.end() + } + + agent.use(traces => { + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.header') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.header'], 'hdr-0110000010-6431a3e6-3-98425651') + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.network') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + }).then(done).catch(done) + + axios.get(`http://localhost:${port}/`) + }) + + it('should provide fingerprinting on failed user login track', (done) => { + controller = (req, res) => { + tracer.appsec.trackUserLoginFailureEvent('test_user_id', true, { metakey: 'metaValue' }) + res.end() + } + + agent.use(traces => { + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.header') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.header'], 'hdr-0110000010-6431a3e6-3-98425651') + assert.property(traces[0][0].meta, '_dd.appsec.fp.http.network') + assert.equal(traces[0][0].meta['_dd.appsec.fp.http.network'], 'net-0-0000000000') + }).then(done).catch(done) + + axios.get(`http://localhost:${port}/`) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/remote_config/index.spec.js b/packages/dd-trace/test/appsec/remote_config/index.spec.js index c3da43a17c0..dbd710d6a4e 100644 --- a/packages/dd-trace/test/appsec/remote_config/index.spec.js +++ b/packages/dd-trace/test/appsec/remote_config/index.spec.js @@ -286,6 +286,12 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) @@ -322,6 +328,12 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) @@ -360,6 +372,12 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, true) expect(rc.updateCapabilities) @@ -393,6 +411,12 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, true) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, true) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) expect(rc.updateCapabilities) .to.not.have.been.calledWith(RemoteConfigCapabilities.ASM_RASP_SSRF) expect(rc.updateCapabilities) @@ -426,6 +450,12 @@ describe('Remote Config index', () => { .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_CUSTOM_BLOCKING_RESPONSE, false) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_TRUSTED_IPS, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_ENDPOINT_FINGERPRINT, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_NETWORK_FINGERPRINT, false) + expect(rc.updateCapabilities) + .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false) expect(rc.updateCapabilities) .to.have.been.calledWithExactly(RemoteConfigCapabilities.ASM_RASP_SSRF, false) expect(rc.updateCapabilities) diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 6fabf747bcf..0860b2c75ac 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -314,20 +314,24 @@ describe('reporter', () => { }) }) - describe('reportSchemas', () => { + describe('reportDerivatives', () => { it('should not call addTags if parameter is undefined', () => { - Reporter.reportSchemas(undefined) + Reporter.reportDerivatives(undefined) expect(span.addTags).not.to.be.called }) it('should call addTags with an empty array', () => { - Reporter.reportSchemas([]) + Reporter.reportDerivatives([]) expect(span.addTags).to.be.calledOnceWithExactly({}) }) it('should call addTags', () => { const schemaValue = [{ key: [8] }] const derivatives = { + '_dd.appsec.fp.http.endpoint': 'endpoint_fingerprint', + '_dd.appsec.fp.http.header': 'header_fingerprint', + '_dd.appsec.fp.http.network': 'network_fingerprint', + '_dd.appsec.fp.session': 'session_fingerprint', '_dd.appsec.s.req.headers': schemaValue, '_dd.appsec.s.req.query': schemaValue, '_dd.appsec.s.req.params': schemaValue, @@ -336,10 +340,14 @@ describe('reporter', () => { 'custom.processor.output': schemaValue } - Reporter.reportSchemas(derivatives) + Reporter.reportDerivatives(derivatives) const schemaEncoded = zlib.gzipSync(JSON.stringify(schemaValue)).toString('base64') expect(span.addTags).to.be.calledOnceWithExactly({ + '_dd.appsec.fp.http.endpoint': 'endpoint_fingerprint', + '_dd.appsec.fp.http.header': 'header_fingerprint', + '_dd.appsec.fp.http.network': 'network_fingerprint', + '_dd.appsec.fp.session': 'session_fingerprint', '_dd.appsec.s.req.headers': schemaEncoded, '_dd.appsec.s.req.query': schemaEncoded, '_dd.appsec.s.req.params': schemaEncoded, diff --git a/packages/dd-trace/test/appsec/sdk/track_event.spec.js b/packages/dd-trace/test/appsec/sdk/track_event.spec.js index acc5db1e905..e3739488b81 100644 --- a/packages/dd-trace/test/appsec/sdk/track_event.spec.js +++ b/packages/dd-trace/test/appsec/sdk/track_event.spec.js @@ -4,6 +4,7 @@ const proxyquire = require('proxyquire') const agent = require('../../plugins/agent') const axios = require('axios') const tracer = require('../../../../../index') +const { LOGIN_SUCCESS, LOGIN_FAILURE } = require('../../../src/appsec/addresses') describe('track_event', () => { describe('Internal API', () => { @@ -14,6 +15,7 @@ describe('track_event', () => { let setUserTags let trackUserLoginSuccessEvent, trackUserLoginFailureEvent, trackCustomEvent, trackEvent let sample + let waf beforeEach(() => { log = { @@ -30,6 +32,10 @@ describe('track_event', () => { sample = sinon.stub() + waf = { + run: sinon.spy() + } + const trackEvents = proxyquire('../../../src/appsec/sdk/track_event', { '../../log': log, './utils': { @@ -40,7 +46,8 @@ describe('track_event', () => { }, '../standalone': { sample - } + }, + '../waf': waf }) trackUserLoginSuccessEvent = trackEvents.trackUserLoginSuccessEvent @@ -49,6 +56,10 @@ describe('track_event', () => { trackEvent = trackEvents.trackEvent }) + afterEach(() => { + sinon.restore() + }) + describe('trackUserLoginSuccessEvent', () => { it('should log warning when passed invalid user', () => { trackUserLoginSuccessEvent(tracer, null, { key: 'value' }) @@ -106,6 +117,16 @@ describe('track_event', () => { '_dd.appsec.events.users.login.success.sdk': 'true' }) }) + + it('should call waf run with login success address', () => { + const user = { id: 'user_id' } + + trackUserLoginSuccessEvent(tracer, user) + sinon.assert.calledOnceWithExactly( + waf.run, + { persistent: { [LOGIN_SUCCESS]: null } } + ) + }) }) describe('trackUserLoginFailureEvent', () => { @@ -182,6 +203,14 @@ describe('track_event', () => { 'appsec.events.users.login.failure.usr.exists': 'true' }) }) + + it('should call waf run with login failure address', () => { + trackUserLoginFailureEvent(tracer, 'user_id') + sinon.assert.calledOnceWithExactly( + waf.run, + { persistent: { [LOGIN_FAILURE]: null } } + ) + }) }) describe('trackCustomEvent', () => { diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index 816b3fe89c6..b0c16647872 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -48,7 +48,7 @@ describe('WAF Manager', () => { sinon.stub(Reporter, 'reportMetrics') sinon.stub(Reporter, 'reportAttack') sinon.stub(Reporter, 'reportWafUpdate') - sinon.stub(Reporter, 'reportSchemas') + sinon.stub(Reporter, 'reportDerivatives') webContext = {} sinon.stub(web, 'getContext').returns(webContext) @@ -404,7 +404,29 @@ describe('WAF Manager', () => { ddwafContext.run.returns(result) wafContextWrapper.run(params) - expect(Reporter.reportSchemas).to.be.calledOnceWithExactly(result.derivatives) + expect(Reporter.reportDerivatives).to.be.calledOnceWithExactly(result.derivatives) + }) + + it('should report fingerprints when ddwafContext returns fingerprints in results derivatives', () => { + const result = { + totalRuntime: 1, + durationExt: 1, + derivatives: { + '_dd.appsec.s.req.body': [8], + '_dd.appsec.fp.http.endpoint': 'http-post-abcdefgh-12345678-abcdefgh', + '_dd.appsec.fp.http.network': 'net-1-0100000000', + '_dd.appsec.fp.http.headers': 'hdr-0110000110-abcdefgh-5-12345678' + } + } + + ddwafContext.run.returns(result) + + wafContextWrapper.run({ + persistent: { + 'server.request.body': 'foo' + } + }) + sinon.assert.calledOnceWithExactly(Reporter.reportDerivatives, result.derivatives) }) }) }) diff --git a/packages/dd-trace/test/plugins/externals.json b/packages/dd-trace/test/plugins/externals.json index e0216047fa4..78373b16daa 100644 --- a/packages/dd-trace/test/plugins/externals.json +++ b/packages/dd-trace/test/plugins/externals.json @@ -341,6 +341,10 @@ { "name": "express", "versions": [">=4.16.2"] + }, + { + "name": "body-parser", + "versions": ["1.20.1"] } ], "pg": [ From 2d175d30d5ad7291c238819116a07f003074316b Mon Sep 17 00:00:00 2001 From: Attila Szegedi Date: Tue, 8 Oct 2024 17:16:39 +0200 Subject: [PATCH 24/27] Keep a profiling context object in spans (#4763) Allows for cheapest sampling context updates on async context switches and opens the path for profiling custom context. --- .../dd-trace/src/profiling/profilers/wall.js | 126 ++++++++---------- 1 file changed, 57 insertions(+), 69 deletions(-) diff --git a/packages/dd-trace/src/profiling/profilers/wall.js b/packages/dd-trace/src/profiling/profilers/wall.js index 39af4ca2bfc..3d7041cfecf 100644 --- a/packages/dd-trace/src/profiling/profilers/wall.js +++ b/packages/dd-trace/src/profiling/profilers/wall.js @@ -20,7 +20,7 @@ const enterCh = dc.channel('dd-trace:storage:enter') const spanFinishCh = dc.channel('dd-trace:span:finish') const profilerTelemetryMetrics = telemetryMetrics.manager.namespace('profilers') -const MemoizedWebTags = Symbol('NativeWallProfiler.MemoizedWebTags') +const ProfilingContext = Symbol('NativeWallProfiler.ProfilingContext') let kSampleCount @@ -44,38 +44,6 @@ function endpointNameFromTags (tags) { ].filter(v => v).join(' ') } -function getWebTags (startedSpans, i, span) { - // Are web tags for this span already memoized? - const memoizedWebTags = span[MemoizedWebTags] - if (memoizedWebTags !== undefined) { - return memoizedWebTags - } - // No, we'll have to memoize a new value - function memoize (tags) { - span[MemoizedWebTags] = tags - return tags - } - // Is this span itself a web span? - const context = span.context() - const tags = context._tags - if (isWebServerSpan(tags)) { - return memoize(tags) - } - // It isn't. Get parent's web tags (memoize them too recursively.) - // There might be several webspans, for example with next.js, http plugin creates the first span - // and then next.js plugin creates a child span, and this child span has the correct endpoint - // information. That's why we always use the tags of the closest ancestor web span. - const parentId = context._parentId - while (--i >= 0) { - const ispan = startedSpans[i] - if (ispan.context()._spanId === parentId) { - return memoize(getWebTags(startedSpans, i, ispan)) - } - } - // Local root span with no web span - return memoize(null) -} - let channelsActivated = false function ensureChannelsActivated () { if (channelsActivated) return @@ -184,14 +152,10 @@ class NativeWallProfiler { }) if (this._withContexts) { - this._currentContext = {} - this._pprof.time.setContext(this._currentContext) + this._setNewContext() if (this._captureSpanData) { this._profilerState = this._pprof.time.getState() - this._lastSpan = undefined - this._lastStartedSpans = undefined - this._lastWebTags = undefined this._lastSampleCount = 0 beforeCh.subscribe(this._enter) @@ -209,51 +173,78 @@ class NativeWallProfiler { const sampleCount = this._profilerState[kSampleCount] if (sampleCount !== this._lastSampleCount) { this._lastSampleCount = sampleCount - const context = this._currentContext - this._currentContext = {} - this._pprof.time.setContext(this._currentContext) + const context = this._currentContext.ref + this._setNewContext() this._updateContext(context) } const span = getActiveSpan() - if (span) { + this._currentContext.ref = span ? this._getProfilingContext(span) : {} + } + + _getProfilingContext (span) { + let profilingContext = span[ProfilingContext] + if (profilingContext === undefined) { const context = span.context() - this._lastSpan = span const startedSpans = getStartedSpans(context) - this._lastStartedSpans = startedSpans + + let spanId + let rootSpanId + if (this._codeHotspotsEnabled) { + spanId = context._spanId + rootSpanId = startedSpans.length ? startedSpans[0].context()._spanId : context._spanId + } + + let webTags if (this._endpointCollectionEnabled) { - this._lastWebTags = getWebTags(startedSpans, startedSpans.length, span) + const tags = context._tags + if (isWebServerSpan(tags)) { + webTags = tags + } else { + // Get parent's context's web tags + const parentId = context._parentId + for (let i = startedSpans.length; --i >= 0;) { + const ispan = startedSpans[i] + if (ispan.context()._spanId === parentId) { + webTags = this._getProfilingContext(ispan).webTags + break + } + } + } } - } else { - this._lastStartedSpans = undefined - this._lastSpan = undefined - this._lastWebTags = undefined + + profilingContext = { spanId, rootSpanId, webTags } + span[ProfilingContext] = profilingContext } + return profilingContext + } + + _setNewContext () { + this._pprof.time.setContext( + this._currentContext = { + ref: {} + } + ) } _updateContext (context) { - if (!this._lastSpan) { - return + if (typeof context.spanId === 'object') { + context.spanId = context.spanId.toString(10) } - if (this._codeHotspotsEnabled) { - context.spanId = this._lastSpan.context().toSpanId() - const rootSpan = this._lastStartedSpans[0] - if (rootSpan) { - context.rootSpanId = rootSpan.context().toSpanId() - } + if (typeof context.rootSpanId === 'object') { + context.rootSpanId = context.rootSpanId.toString(10) } - if (this._lastWebTags) { - context.webTags = this._lastWebTags + if (context.webTags !== undefined && context.endpoint === undefined) { // endpoint may not be determined yet, but keep it as fallback // if tags are not available anymore during serialization - context.endpoint = endpointNameFromTags(this._lastWebTags) + context.endpoint = endpointNameFromTags(context.webTags) } } _spanFinished (span) { - if (span[MemoizedWebTags]) { - span[MemoizedWebTags] = undefined + if (span[ProfilingContext] !== undefined) { + span[ProfilingContext] = undefined } } @@ -288,9 +279,6 @@ class NativeWallProfiler { enterCh.unsubscribe(this._enter) spanFinishCh.unsubscribe(this._spanFinished) this._profilerState = undefined - this._lastSpan = undefined - this._lastStartedSpans = undefined - this._lastWebTags = undefined } this._started = false } @@ -313,20 +301,20 @@ class NativeWallProfiler { const labels = { ...getThreadLabels() } - const { context: { spanId, rootSpanId, webTags, endpoint }, timestamp } = context + const { context: { ref: { spanId, rootSpanId, webTags, endpoint } }, timestamp } = context if (this._timelineEnabled) { // Incoming timestamps are in microseconds, we emit nanos. labels[END_TIMESTAMP_LABEL] = timestamp * 1000n } - if (spanId) { + if (spanId !== undefined) { labels[SPAN_ID_LABEL] = spanId } - if (rootSpanId) { + if (rootSpanId !== undefined) { labels[LOCAL_ROOT_SPAN_ID_LABEL] = rootSpanId } - if (webTags && Object.keys(webTags).length !== 0) { + if (webTags !== undefined && Object.keys(webTags).length !== 0) { labels['trace endpoint'] = endpointNameFromTags(webTags) } else if (endpoint) { // fallback to endpoint computed when sample was taken From 5eea208392edfa3abce1f2a52fe3f06cfb3d0037 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Wed, 9 Oct 2024 09:22:31 +0200 Subject: [PATCH 25/27] [test visibility] Add option to automatically report logs within tests when using `winston` (#4762) --- .../automatic-log-submission.spec.js | 207 ++++++++++++++++++ integration-tests/ci-visibility-intake.js | 9 + .../automatic-log-submission.feature | 4 + .../support/logger.js | 10 + .../support/steps.js | 14 ++ .../support/sum.js | 6 + .../automatic-log-submission-test.js | 13 ++ .../automatic-log-submission/config-jest.js | 8 + .../automatic-log-submission/logger.js | 10 + .../automatic-log-submission/sum.js | 6 + integration-tests/selenium/selenium.spec.js | 2 +- packages/datadog-instrumentations/src/jest.js | 3 +- .../datadog-instrumentations/src/winston.js | 22 ++ .../log-submission/log-submission-plugin.js | 53 +++++ packages/dd-trace/src/config.js | 5 +- packages/dd-trace/src/plugin_manager.js | 6 +- packages/dd-trace/src/plugins/ci_plugin.js | 2 + packages/dd-trace/src/plugins/log_plugin.js | 2 +- packages/dd-trace/src/proxy.js | 12 + 19 files changed, 388 insertions(+), 6 deletions(-) create mode 100644 integration-tests/automatic-log-submission.spec.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature create mode 100644 integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission/config-jest.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission/logger.js create mode 100644 integration-tests/ci-visibility/automatic-log-submission/sum.js create mode 100644 packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js diff --git a/integration-tests/automatic-log-submission.spec.js b/integration-tests/automatic-log-submission.spec.js new file mode 100644 index 00000000000..eade717dcf1 --- /dev/null +++ b/integration-tests/automatic-log-submission.spec.js @@ -0,0 +1,207 @@ +'use strict' + +const { exec } = require('child_process') + +const { assert } = require('chai') +const getPort = require('get-port') + +const { + createSandbox, + getCiVisAgentlessConfig, + getCiVisEvpProxyConfig +} = require('./helpers') +const { FakeCiVisIntake } = require('./ci-visibility-intake') +const webAppServer = require('./ci-visibility/web-app-server') +const { NODE_MAJOR } = require('../version') + +const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' + +describe('test visibility automatic log submission', () => { + let sandbox, cwd, receiver, childProcess, webAppPort + let testOutput = '' + + before(async () => { + sandbox = await createSandbox([ + 'mocha', + `@cucumber/cucumber@${cucumberVersion}`, + 'jest', + 'winston', + 'chai@4' + ], true) + cwd = sandbox.folder + webAppPort = await getPort() + webAppServer.listen(webAppPort) + }) + + after(async () => { + await sandbox.remove() + await new Promise(resolve => webAppServer.close(resolve)) + }) + + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) + + afterEach(async () => { + testOutput = '' + childProcess.kill() + await receiver.stop() + }) + + const testFrameworks = [ + { + name: 'mocha', + command: 'mocha ./ci-visibility/automatic-log-submission/automatic-log-submission-test.js' + }, + { + name: 'jest', + command: 'node ./node_modules/jest/bin/jest --config ./ci-visibility/automatic-log-submission/config-jest.js' + }, + { + name: 'cucumber', + command: './node_modules/.bin/cucumber-js ci-visibility/automatic-log-submission-cucumber/*.feature' + } + ] + + testFrameworks.forEach(({ name, command }) => { + context(`with ${name}`, () => { + it('can automatically submit logs', (done) => { + let logIds, testIds + + const logsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.includes('/api/v2/logs'), payloads => { + payloads.forEach(({ headers }) => { + assert.equal(headers['dd-api-key'], '1') + }) + const logMessages = payloads.flatMap(({ logMessage }) => logMessage) + const [url] = payloads.flatMap(({ url }) => url) + + assert.equal(url, '/api/v2/logs?ddsource=winston&service=my-service') + assert.equal(logMessages.length, 2) + + logMessages.forEach(({ dd, level }) => { + assert.equal(level, 'info') + assert.equal(dd.service, 'my-service') + assert.hasAllKeys(dd, ['trace_id', 'span_id', 'service']) + }) + + assert.includeMembers(logMessages.map(({ message }) => message), [ + 'Hello simple log!', + 'sum function being called' + ]) + + logIds = { + logSpanId: logMessages[0].dd.span_id, + logTraceId: logMessages[0].dd.trace_id + } + }) + + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const testEventContent = events.find(event => event.type === 'test').content + + testIds = { + testSpanId: testEventContent.span_id.toString(), + testTraceId: testEventContent.trace_id.toString() + } + }) + + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_ENABLED: '1', + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_API_KEY: '1', + DD_SERVICE: 'my-service' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + Promise.all([logsPromise, eventsPromise]).then(() => { + const { logSpanId, logTraceId } = logIds + const { testSpanId, testTraceId } = testIds + assert.include(testOutput, 'Hello simple log!') + assert.include(testOutput, 'sum function being called') + // cucumber has `cucumber.step`, and that's the active span, not the test. + // logs are queried by trace id, so it should be OK + if (name !== 'cucumber') { + assert.include(testOutput, `"span_id":"${testSpanId}"`) + assert.equal(logSpanId, testSpanId) + } + assert.include(testOutput, `"trace_id":"${testTraceId}"`) + assert.equal(logTraceId, testTraceId) + done() + }).catch(done) + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not submit logs when DD_AGENTLESS_LOG_SUBMISSION_ENABLED is not set', (done) => { + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_SERVICE: 'my-service' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + assert.include(testOutput, 'Hello simple log!') + assert.notInclude(testOutput, 'span_id') + done() + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + + it('does not submit logs when DD_AGENTLESS_LOG_SUBMISSION_ENABLED is set but DD_API_KEY is not', (done) => { + childProcess = exec(command, + { + cwd, + env: { + ...getCiVisEvpProxyConfig(receiver.port), + DD_AGENTLESS_LOG_SUBMISSION_ENABLED: '1', + DD_AGENTLESS_LOG_SUBMISSION_URL: `http://localhost:${receiver.port}`, + DD_SERVICE: 'my-service', + DD_TRACE_DEBUG: '1', + DD_TRACE_LOG_LEVEL: 'warn', + DD_API_KEY: '' + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + assert.include(testOutput, 'Hello simple log!') + assert.include(testOutput, 'no automatic log submission will be performed') + done() + }) + + childProcess.stdout.on('data', (chunk) => { + testOutput += chunk.toString() + }) + childProcess.stderr.on('data', (chunk) => { + testOutput += chunk.toString() + }) + }) + }) + }) +}) diff --git a/integration-tests/ci-visibility-intake.js b/integration-tests/ci-visibility-intake.js index 5096efaba42..c133a7a31fe 100644 --- a/integration-tests/ci-visibility-intake.js +++ b/integration-tests/ci-visibility-intake.js @@ -208,6 +208,15 @@ class FakeCiVisIntake extends FakeAgent { }) }) + app.post('/api/v2/logs', express.json(), (req, res) => { + res.status(200).send('OK') + this.emit('message', { + headers: req.headers, + url: req.url, + logMessage: req.body + }) + }) + return new Promise((resolve, reject) => { const timeoutObj = setTimeout(() => { reject(new Error('Intake timed out starting up')) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature b/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature new file mode 100644 index 00000000000..bcce6b75bea --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/automatic-log-submission.feature @@ -0,0 +1,4 @@ +Feature: Automatic Log Submission + Scenario: Run Automatic Log Submission + When we run a test + Then I should have made a log diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js new file mode 100644 index 00000000000..5480f1ee574 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/logger.js @@ -0,0 +1,10 @@ +const { createLogger, format, transports } = require('winston') + +module.exports = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Console() + ] +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js new file mode 100644 index 00000000000..2d1bdb4e906 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/steps.js @@ -0,0 +1,14 @@ +const { expect } = require('chai') +const { When, Then } = require('@cucumber/cucumber') + +const logger = require('./logger') +const sum = require('./sum') + +Then('I should have made a log', async function () { + expect(true).to.equal(true) + expect(sum(1, 2)).to.equal(3) +}) + +When('we run a test', async function () { + logger.log('info', 'Hello simple log!') +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js new file mode 100644 index 00000000000..cce61142972 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission-cucumber/support/sum.js @@ -0,0 +1,6 @@ +const logger = require('./logger') + +module.exports = function (a, b) { + logger.log('info', 'sum function being called') + return a + b +} diff --git a/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js b/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js new file mode 100644 index 00000000000..cfc60b8d3b0 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/automatic-log-submission-test.js @@ -0,0 +1,13 @@ +const { expect } = require('chai') + +const logger = require('./logger') +const sum = require('./sum') + +describe('test', () => { + it('should return true', () => { + logger.log('info', 'Hello simple log!') + + expect(true).to.be.true + expect(sum(1, 2)).to.equal(3) + }) +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission/config-jest.js b/integration-tests/ci-visibility/automatic-log-submission/config-jest.js new file mode 100644 index 00000000000..56afa0d36db --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/config-jest.js @@ -0,0 +1,8 @@ +module.exports = { + projects: [], + testPathIgnorePatterns: ['/node_modules/'], + cache: false, + testMatch: [ + '**/ci-visibility/automatic-log-submission/automatic-log-submission-*' + ] +} diff --git a/integration-tests/ci-visibility/automatic-log-submission/logger.js b/integration-tests/ci-visibility/automatic-log-submission/logger.js new file mode 100644 index 00000000000..5480f1ee574 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/logger.js @@ -0,0 +1,10 @@ +const { createLogger, format, transports } = require('winston') + +module.exports = createLogger({ + level: 'info', + exitOnError: false, + format: format.json(), + transports: [ + new transports.Console() + ] +}) diff --git a/integration-tests/ci-visibility/automatic-log-submission/sum.js b/integration-tests/ci-visibility/automatic-log-submission/sum.js new file mode 100644 index 00000000000..cce61142972 --- /dev/null +++ b/integration-tests/ci-visibility/automatic-log-submission/sum.js @@ -0,0 +1,6 @@ +const logger = require('./logger') + +module.exports = function (a, b) { + logger.log('info', 'sum function being called') + return a + b +} diff --git a/integration-tests/selenium/selenium.spec.js b/integration-tests/selenium/selenium.spec.js index a95acb6aaa2..50fc9d19568 100644 --- a/integration-tests/selenium/selenium.spec.js +++ b/integration-tests/selenium/selenium.spec.js @@ -18,7 +18,7 @@ const { } = require('../../packages/dd-trace/src/plugins/util/test') const { NODE_MAJOR } = require('../../version') -const cucumberVersion = NODE_MAJOR <= 16 ? '9' : '10' +const cucumberVersion = NODE_MAJOR <= 16 ? '9' : 'latest' const webAppServer = require('../ci-visibility/web-app-server') diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index e2baf3f9d42..e006f311dc3 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -850,7 +850,8 @@ addHook({ }, jestConfigSyncWrapper) const LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE = [ - 'selenium-webdriver' + 'selenium-webdriver', + 'winston' ] function shouldBypassJestRequireEngine (moduleName) { diff --git a/packages/datadog-instrumentations/src/winston.js b/packages/datadog-instrumentations/src/winston.js index dbd91026bf2..9b9c4e811aa 100644 --- a/packages/datadog-instrumentations/src/winston.js +++ b/packages/datadog-instrumentations/src/winston.js @@ -8,6 +8,18 @@ const shimmer = require('../../datadog-shimmer') const patched = new WeakSet() +// Test Visibility log submission channels +const configureCh = channel('ci:log-submission:winston:configure') +const addTransport = channel('ci:log-submission:winston:add-transport') + +addHook({ name: 'winston', file: 'lib/winston/transports/index.js', versions: ['>=3'] }, transportsPackage => { + if (configureCh.hasSubscribers) { + configureCh.publish(transportsPackage.Http) + } + + return transportsPackage +}) + addHook({ name: 'winston', file: 'lib/winston/logger.js', versions: ['>=3'] }, Logger => { const logCh = channel('apm:winston:log') shimmer.wrap(Logger.prototype, 'write', write => { @@ -20,6 +32,16 @@ addHook({ name: 'winston', file: 'lib/winston/logger.js', versions: ['>=3'] }, L return write.apply(this, arguments) } }) + + shimmer.wrap(Logger.prototype, 'configure', configure => function () { + const configureResponse = configure.apply(this, arguments) + // After the original `configure`, because it resets transports + if (addTransport.hasSubscribers) { + addTransport.publish(this) + } + return configureResponse + }) + return Logger }) diff --git a/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js b/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js new file mode 100644 index 00000000000..aa437f4cd87 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/log-submission/log-submission-plugin.js @@ -0,0 +1,53 @@ +const Plugin = require('../../plugins/plugin') +const log = require('../../log') + +function getWinstonLogSubmissionParameters (config) { + const { site, service } = config + + const defaultParameters = { + host: `http-intake.logs.${site}`, + path: `/api/v2/logs?ddsource=winston&service=${service}`, + ssl: true, + headers: { + 'DD-API-KEY': process.env.DD_API_KEY + } + } + + if (!process.env.DD_AGENTLESS_LOG_SUBMISSION_URL) { + return defaultParameters + } + + try { + const url = new URL(process.env.DD_AGENTLESS_LOG_SUBMISSION_URL) + return { + host: url.hostname, + port: url.port, + ssl: url.protocol === 'https:', + path: defaultParameters.path, + headers: defaultParameters.headers + } + } catch (e) { + log.error('Could not parse DD_AGENTLESS_LOG_SUBMISSION_URL') + return defaultParameters + } +} + +class LogSubmissionPlugin extends Plugin { + static get id () { + return 'log-submission' + } + + constructor (...args) { + super(...args) + + this.addSub('ci:log-submission:winston:configure', (httpClass) => { + this.HttpClass = httpClass + }) + + this.addSub('ci:log-submission:winston:add-transport', (logger) => { + logger.add(new this.HttpClass(getWinstonLogSubmissionParameters(this.config))) + }) + } +} + +module.exports = LogSubmissionPlugin diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index dc5bb524d1a..69005edd369 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -498,6 +498,7 @@ class Config { this._setValue(defaults, 'isIntelligentTestRunnerEnabled', false) this._setValue(defaults, 'isManualApiEnabled', false) this._setValue(defaults, 'ciVisibilityTestSessionName', '') + this._setValue(defaults, 'ciVisAgentlessLogSubmissionEnabled', false) this._setValue(defaults, 'logInjection', false) this._setValue(defaults, 'lookup', undefined) this._setValue(defaults, 'memcachedCommandEnabled', false) @@ -1035,7 +1036,8 @@ class Config { DD_CIVISIBILITY_EARLY_FLAKE_DETECTION_ENABLED, DD_CIVISIBILITY_FLAKY_RETRY_ENABLED, DD_CIVISIBILITY_FLAKY_RETRY_COUNT, - DD_TEST_SESSION_NAME + DD_TEST_SESSION_NAME, + DD_AGENTLESS_LOG_SUBMISSION_ENABLED } = process.env if (DD_CIVISIBILITY_AGENTLESS_URL) { @@ -1052,6 +1054,7 @@ class Config { this._setBoolean(calc, 'isIntelligentTestRunnerEnabled', isTrue(this._isCiVisibilityItrEnabled())) this._setBoolean(calc, 'isManualApiEnabled', !isFalse(this._isCiVisibilityManualApiEnabled())) this._setString(calc, 'ciVisibilityTestSessionName', DD_TEST_SESSION_NAME) + this._setBoolean(calc, 'ciVisAgentlessLogSubmissionEnabled', isTrue(DD_AGENTLESS_LOG_SUBMISSION_ENABLED)) } this._setString(calc, 'dogstatsd.hostname', this._getHostname()) this._setBoolean(calc, 'isGitUploadEnabled', diff --git a/packages/dd-trace/src/plugin_manager.js b/packages/dd-trace/src/plugin_manager.js index 80e87ce545e..e9daea9b60b 100644 --- a/packages/dd-trace/src/plugin_manager.js +++ b/packages/dd-trace/src/plugin_manager.js @@ -137,7 +137,8 @@ module.exports = class PluginManager { dsmEnabled, clientIpEnabled, memcachedCommandEnabled, - ciVisibilityTestSessionName + ciVisibilityTestSessionName, + ciVisAgentlessLogSubmissionEnabled } = this._tracerConfig const sharedConfig = { @@ -147,7 +148,8 @@ module.exports = class PluginManager { site, url, headers: headerTags || [], - ciVisibilityTestSessionName + ciVisibilityTestSessionName, + ciVisAgentlessLogSubmissionEnabled } if (logInjection !== undefined) { diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index b86d20d5760..d4c9f32bc68 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -100,7 +100,9 @@ module.exports = class CiPlugin extends Plugin { ...testSessionSpanMetadata } }) + // TODO: add telemetry tag when we can add `is_agentless_log_submission_enabled` for agentless log submission this.telemetry.ciVisEvent(TELEMETRY_EVENT_CREATED, 'session') + this.testModuleSpan = this.tracer.startSpan(`${this.constructor.id}.test_module`, { childOf: this.testSessionSpan, tags: { diff --git a/packages/dd-trace/src/plugins/log_plugin.js b/packages/dd-trace/src/plugins/log_plugin.js index 353008a9e02..b0812ea46d3 100644 --- a/packages/dd-trace/src/plugins/log_plugin.js +++ b/packages/dd-trace/src/plugins/log_plugin.js @@ -54,7 +54,7 @@ module.exports = class LogPlugin extends Plugin { configure (config) { return super.configure({ ...config, - enabled: config.enabled && config.logInjection + enabled: config.enabled && (config.logInjection || config.ciVisAgentlessLogSubmissionEnabled) }) } } diff --git a/packages/dd-trace/src/proxy.js b/packages/dd-trace/src/proxy.js index d7ce0538f39..b8916b205d4 100644 --- a/packages/dd-trace/src/proxy.js +++ b/packages/dd-trace/src/proxy.js @@ -162,6 +162,18 @@ class Tracer extends NoopProxy { this._testApiManualPlugin.configure({ ...config, enabled: true }) } } + if (config.ciVisAgentlessLogSubmissionEnabled) { + if (process.env.DD_API_KEY) { + const LogSubmissionPlugin = require('./ci-visibility/log-submission/log-submission-plugin') + const automaticLogPlugin = new LogSubmissionPlugin(this) + automaticLogPlugin.configure({ ...config, enabled: true }) + } else { + log.warn( + 'DD_AGENTLESS_LOG_SUBMISSION_ENABLED is set, ' + + 'but DD_API_KEY is undefined, so no automatic log submission will be performed.' + ) + } + } } catch (e) { log.error(e) } From 60529442d2184aa8f05ed906c8f834e51849ad26 Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Wed, 9 Oct 2024 11:51:27 +0200 Subject: [PATCH 26/27] Use static vulnerability hash source when the cookie name is too long (#4764) --- docs/test.ts | 2 + index.d.ts | 7 +- .../appsec/iast/analyzers/cookie-analyzer.js | 14 ++- .../dd-trace/src/appsec/iast/iast-plugin.js | 2 +- packages/dd-trace/src/config.js | 4 + .../iast/analyzers/cookie-analyzer.spec.js | 110 ++++++++++++++++++ .../insecure-cookie-analyzer.spec.js | 14 +++ .../no-httponly-cookie-analyzer.spec.js | 14 +++ .../no-samesite-cookie-analyzer.spec.js | 13 +++ packages/dd-trace/test/appsec/iast/utils.js | 8 +- packages/dd-trace/test/config.spec.js | 13 +++ 11 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js diff --git a/docs/test.ts b/docs/test.ts index e948e4ff4dd..3ec8d20bb68 100644 --- a/docs/test.ts +++ b/docs/test.ts @@ -128,6 +128,7 @@ tracer.init({ }, iast: { enabled: true, + cookieFilterPattern: '.*', requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 30, @@ -143,6 +144,7 @@ tracer.init({ experimental: { iast: { enabled: true, + cookieFilterPattern: '.*', requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 30, diff --git a/index.d.ts b/index.d.ts index bc17ef2dad5..f7ab601fe51 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1739,7 +1739,6 @@ declare namespace tracer { * on the tracer. */ interface pino extends Integration {} - /** * This plugin automatically patches the [protobufjs](https://protobufjs.github.io/protobuf.js/) * to collect protobuf message schemas when Datastreams Monitoring is enabled. @@ -2160,6 +2159,12 @@ declare namespace tracer { */ maxContextOperations?: number, + /** + * Defines the pattern to ignore cookie names in the vulnerability hash calculation + * @default ".{32,}" + */ + cookieFilterPattern?: string, + /** * Whether to enable vulnerability deduplication */ diff --git a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js index e82fe65ef74..2b125b88403 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/cookie-analyzer.js @@ -2,6 +2,7 @@ const Analyzer = require('./vulnerability-analyzer') const { getNodeModulesPaths } = require('../path-line') +const iastLog = require('../iast-log') const EXCLUDED_PATHS = getNodeModulesPaths('express/lib/response.js') @@ -11,7 +12,14 @@ class CookieAnalyzer extends Analyzer { this.propertyToBeSafe = propertyToBeSafe.toLowerCase() } - onConfigure () { + onConfigure (config) { + try { + this.cookieFilterRegExp = new RegExp(config.iast.cookieFilterPattern) + } catch { + iastLog.error('Invalid regex in cookieFilterPattern') + this.cookieFilterRegExp = /.{32,}/ + } + this.addSub( { channelName: 'datadog:iast:set-cookie', moduleName: 'http' }, (cookieInfo) => this.analyze(cookieInfo) @@ -28,6 +36,10 @@ class CookieAnalyzer extends Analyzer { } _createHashSource (type, evidence, location) { + if (typeof evidence.value === 'string' && evidence.value.match(this.cookieFilterRegExp)) { + return 'FILTERED_' + this._type + } + return `${type}:${evidence.value}` } diff --git a/packages/dd-trace/src/appsec/iast/iast-plugin.js b/packages/dd-trace/src/appsec/iast/iast-plugin.js index 96759a530e2..5eb6e00410d 100644 --- a/packages/dd-trace/src/appsec/iast/iast-plugin.js +++ b/packages/dd-trace/src/appsec/iast/iast-plugin.js @@ -127,7 +127,7 @@ class IastPlugin extends Plugin { config = { enabled: config } } if (config.enabled && !this.configured) { - this.onConfigure() + this.onConfigure(config.tracerConfig) this.configured = true } diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 69005edd369..c6cd23945ba 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -478,6 +478,7 @@ class Config { this._setValue(defaults, 'gitMetadataEnabled', true) this._setValue(defaults, 'headerTags', []) this._setValue(defaults, 'hostname', '127.0.0.1') + this._setValue(defaults, 'iast.cookieFilterPattern', '.{32,}') this._setValue(defaults, 'iast.deduplicationEnabled', true) this._setValue(defaults, 'iast.enabled', false) this._setValue(defaults, 'iast.maxConcurrentRequests', 2) @@ -582,6 +583,7 @@ class Config { DD_EXPERIMENTAL_APPSEC_STANDALONE_ENABLED, DD_EXPERIMENTAL_PROFILING_ENABLED, JEST_WORKER_ID, + DD_IAST_COOKIE_FILTER_PATTERN, DD_IAST_DEDUPLICATION_ENABLED, DD_IAST_ENABLED, DD_IAST_MAX_CONCURRENT_REQUESTS, @@ -717,6 +719,7 @@ class Config { this._setBoolean(env, 'gitMetadataEnabled', DD_TRACE_GIT_METADATA_ENABLED) this._setArray(env, 'headerTags', DD_TRACE_HEADER_TAGS) this._setString(env, 'hostname', coalesce(DD_AGENT_HOST, DD_TRACE_AGENT_HOSTNAME)) + this._setString(env, 'iast.cookieFilterPattern', DD_IAST_COOKIE_FILTER_PATTERN) this._setBoolean(env, 'iast.deduplicationEnabled', DD_IAST_DEDUPLICATION_ENABLED) this._setBoolean(env, 'iast.enabled', DD_IAST_ENABLED) this._setValue(env, 'iast.maxConcurrentRequests', maybeInt(DD_IAST_MAX_CONCURRENT_REQUESTS)) @@ -885,6 +888,7 @@ class Config { this._optsUnprocessed.flushMinSpans = options.flushMinSpans this._setArray(opts, 'headerTags', options.headerTags) this._setString(opts, 'hostname', options.hostname) + this._setString(opts, 'iast.cookieFilterPattern', options.iast?.cookieFilterPattern) this._setBoolean(opts, 'iast.deduplicationEnabled', options.iast && options.iast.deduplicationEnabled) this._setBoolean(opts, 'iast.enabled', options.iast && (options.iast === true || options.iast.enabled === true)) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js new file mode 100644 index 00000000000..ba9c114a5c1 --- /dev/null +++ b/packages/dd-trace/test/appsec/iast/analyzers/cookie-analyzer.spec.js @@ -0,0 +1,110 @@ +'use strict' + +const { assert } = require('chai') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') +const Config = require('../../../../src/config') + +describe('CookieAnalyzer', () => { + const VULNERABILITY_TYPE = 'VULN_TYPE' + + it('should extends Analyzer', () => { + assert.isTrue(Analyzer.isPrototypeOf(CookieAnalyzer)) + }) + + describe('_createHashSource', () => { + let cookieAnalyzer + + beforeEach(() => { + cookieAnalyzer = new CookieAnalyzer(VULNERABILITY_TYPE, 'prop') + }) + + describe('default config', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ iast: true })) + }) + + it('should create hash from vulnerability type and not long enough evidence value', () => { + const evidence = { + value: '0'.repeat(31) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: '0'.repeat(32) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + + describe('custom cookieFilterPattern', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ + iast: { + enabled: true, + cookieFilterPattern: '^filtered$' + } + })) + }) + + it('should create hash from vulnerability with the default pattern', () => { + const evidence = { + value: 'notfiltered' + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: 'filtered' + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + + describe('invalid cookieFilterPattern maintains default behaviour', () => { + beforeEach(() => { + cookieAnalyzer.onConfigure(new Config({ + iast: { + enabled: true, + cookieFilterPattern: '(' + } + })) + }) + + it('should create hash from vulnerability type and not long enough evidence value', () => { + const evidence = { + value: '0'.repeat(31) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`${VULNERABILITY_TYPE}:${evidence.value}`)) + }) + + it('should create different hash from vulnerability type and long evidence value', () => { + const evidence = { + value: '0'.repeat(32) + } + + const vulnerability = cookieAnalyzer._createVulnerability(VULNERABILITY_TYPE, evidence, null, {}) + + assert.equal(vulnerability.hash, cookieAnalyzer._createHash(`FILTERED_${VULNERABILITY_TYPE}`)) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js index fbb3454c27e..af4bd911325 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/insecure-cookie-analyzer.spec.js @@ -3,12 +3,20 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { INSECURE_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const insecureCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/insecure-cookie-analyzer') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') + const analyzer = new Analyzer() describe('insecure cookie analyzer', () => { it('Expected vulnerability identifier', () => { expect(INSECURE_COOKIE).to.be.equals('INSECURE_COOKIE') }) + + it('InsecureCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(insecureCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -43,6 +51,12 @@ describe('insecure cookie analyzer', () => { res.setHeader('set-cookie', ['key=value; HttpOnly', 'key2=value2; Secure']) }, INSECURE_COOKIE, 1) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, INSECURE_COOKIE, 1, undefined, undefined, + 'Should be detected as the same INSECURE_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=value; Secure') }, INSECURE_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js index 3c9ed1bae19..743db43097c 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/no-httponly-cookie-analyzer.spec.js @@ -3,6 +3,9 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { NO_HTTPONLY_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const noHttponlyCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/no-httponly-cookie-analyzer') + const analyzer = new Analyzer() describe('no HttpOnly cookie analyzer', () => { @@ -10,6 +13,10 @@ describe('no HttpOnly cookie analyzer', () => { expect(NO_HTTPONLY_COOKIE).to.be.equals('NO_HTTPONLY_COOKIE') }) + it('NoHttponlyCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(noHttponlyCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -18,6 +25,7 @@ describe('no HttpOnly cookie analyzer', () => { maxConcurrentRequests: 1, maxContextOperations: 1 } + prepareTestServerForIast('no HttpOnly cookie analyzer', (testThatRequestHasVulnerability, testThatRequestHasNoVulnerability) => { testThatRequestHasVulnerability((req, res) => { @@ -47,6 +55,12 @@ describe('no HttpOnly cookie analyzer', () => { res.setHeader('set-cookie', ['key=value; HttpOnly', 'key2=value2; Secure']) }, NO_HTTPONLY_COOKIE, 1) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, NO_HTTPONLY_COOKIE, 1, undefined, undefined, + 'Should be detected as the same NO_HTTPONLY_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=value; HttpOnly') }, NO_HTTPONLY_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js b/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js index 03be8280795..0d7b1f26dc9 100644 --- a/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js +++ b/packages/dd-trace/test/appsec/iast/analyzers/no-samesite-cookie-analyzer.spec.js @@ -3,6 +3,9 @@ const { prepareTestServerForIast } = require('../utils') const Analyzer = require('../../../../src/appsec/iast/analyzers/vulnerability-analyzer') const { NO_SAMESITE_COOKIE } = require('../../../../src/appsec/iast/vulnerabilities') +const CookieAnalyzer = require('../../../../src/appsec/iast/analyzers/cookie-analyzer') +const noSamesiteCookieAnalyzer = require('../../../../src/appsec/iast/analyzers/no-samesite-cookie-analyzer') + const analyzer = new Analyzer() describe('no SameSite cookie analyzer', () => { @@ -10,6 +13,10 @@ describe('no SameSite cookie analyzer', () => { expect(NO_SAMESITE_COOKIE).to.be.equals('NO_SAMESITE_COOKIE') }) + it('NoSamesiteCookieAnalyzer extends CookieAnalyzer', () => { + expect(CookieAnalyzer.isPrototypeOf(noSamesiteCookieAnalyzer.constructor)).to.be.true + }) + // In these test, even when we are having multiple vulnerabilities, all the vulnerabilities // are in the same cookies method, and it is expected to detect both even when the max operations is 1 const iastConfig = { @@ -59,6 +66,12 @@ describe('no SameSite cookie analyzer', () => { res.setHeader('set-cookie', 'key=value; SameSite=strict') }, NO_SAMESITE_COOKIE) + testThatRequestHasVulnerability((req, res) => { + const cookieNamePrefix = '0'.repeat(32) + res.setHeader('set-cookie', [cookieNamePrefix + 'key1=value', cookieNamePrefix + 'key2=value2']) + }, NO_SAMESITE_COOKIE, 1, undefined, undefined, + 'Should be detected as the same NO_SAMESITE_COOKIE vulnerability when the cookie name is long') + testThatRequestHasNoVulnerability((req, res) => { res.setHeader('set-cookie', 'key=') }, NO_SAMESITE_COOKIE) diff --git a/packages/dd-trace/test/appsec/iast/utils.js b/packages/dd-trace/test/appsec/iast/utils.js index 23d9d73260a..2ef5a77ee30 100644 --- a/packages/dd-trace/test/appsec/iast/utils.js +++ b/packages/dd-trace/test/appsec/iast/utils.js @@ -112,9 +112,7 @@ function beforeEachIastTest (iastConfig) { beforeEach(() => { vulnerabilityReporter.clearCache() iast.enable(new Config({ - experimental: { - iast: iastConfig - } + iast: iastConfig })) }) } @@ -249,8 +247,8 @@ function prepareTestServerForIast (description, tests, iastConfig) { return agent.close({ ritmReset: false }) }) - function testThatRequestHasVulnerability (fn, vulnerability, occurrences, cb, makeRequest) { - it(`should have ${vulnerability} vulnerability`, function (done) { + function testThatRequestHasVulnerability (fn, vulnerability, occurrences, cb, makeRequest, description) { + it(description || `should have ${vulnerability} vulnerability`, function (done) { this.timeout(5000) app = fn checkVulnerabilityInRequest(vulnerability, occurrences, cb, makeRequest, config, done) diff --git a/packages/dd-trace/test/config.spec.js b/packages/dd-trace/test/config.spec.js index ca4d8b142d3..ec34d7e71dd 100644 --- a/packages/dd-trace/test/config.spec.js +++ b/packages/dd-trace/test/config.spec.js @@ -305,6 +305,7 @@ describe('Config', () => { { name: 'gitMetadataEnabled', value: true, origin: 'default' }, { name: 'headerTags', value: [], origin: 'default' }, { name: 'hostname', value: '127.0.0.1', origin: 'default' }, + { name: 'iast.cookieFilterPattern', value: '.{32,}', origin: 'default' }, { name: 'iast.deduplicationEnabled', value: true, origin: 'default' }, { name: 'iast.enabled', value: false, origin: 'default' }, { name: 'iast.maxConcurrentRequests', value: 2, origin: 'default' }, @@ -475,6 +476,7 @@ describe('Config', () => { process.env.DD_IAST_REQUEST_SAMPLING = '40' process.env.DD_IAST_MAX_CONCURRENT_REQUESTS = '3' process.env.DD_IAST_MAX_CONTEXT_OPERATIONS = '4' + process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_DEDUPLICATION_ENABLED = false process.env.DD_IAST_REDACTION_ENABLED = false process.env.DD_IAST_REDACTION_NAME_PATTERN = 'REDACTION_NAME_PATTERN' @@ -574,6 +576,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.requestSampling', 40) expect(config).to.have.nested.property('iast.maxConcurrentRequests', 3) expect(config).to.have.nested.property('iast.maxContextOperations', 4) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') @@ -612,6 +615,7 @@ describe('Config', () => { { name: 'experimental.exporter', value: 'log', origin: 'env_var' }, { name: 'experimental.runtimeId', value: true, origin: 'env_var' }, { name: 'hostname', value: 'agent', origin: 'env_var' }, + { name: 'iast.cookieFilterPattern', value: '.*', origin: 'env_var' }, { name: 'iast.deduplicationEnabled', value: false, origin: 'env_var' }, { name: 'iast.enabled', value: true, origin: 'env_var' }, { name: 'iast.maxConcurrentRequests', value: '3', origin: 'env_var' }, @@ -776,6 +780,7 @@ describe('Config', () => { requestSampling: 50, maxConcurrentRequests: 4, maxContextOperations: 5, + cookieFilterPattern: '.*', deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', @@ -841,6 +846,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.requestSampling', 50) expect(config).to.have.nested.property('iast.maxConcurrentRequests', 4) expect(config).to.have.nested.property('iast.maxContextOperations', 5) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.*') expect(config).to.have.nested.property('iast.deduplicationEnabled', false) expect(config).to.have.nested.property('iast.redactionEnabled', false) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') @@ -884,6 +890,7 @@ describe('Config', () => { { name: 'flushInterval', value: 5000, origin: 'code' }, { name: 'flushMinSpans', value: 500, origin: 'code' }, { name: 'hostname', value: 'agent', origin: 'code' }, + { name: 'iast.cookieFilterPattern', value: '.*', origin: 'code' }, { name: 'iast.deduplicationEnabled', value: false, origin: 'code' }, { name: 'iast.enabled', value: true, origin: 'code' }, { name: 'iast.maxConcurrentRequests', value: 4, origin: 'code' }, @@ -1081,6 +1088,7 @@ describe('Config', () => { process.env.DD_API_SECURITY_REQUEST_SAMPLE_RATE = 0.5 process.env.DD_REMOTE_CONFIG_POLL_INTERVAL_SECONDS = 11 process.env.DD_IAST_ENABLED = 'false' + process.env.DD_IAST_COOKIE_FILTER_PATTERN = '.*' process.env.DD_IAST_REDACTION_NAME_PATTERN = 'name_pattern_to_be_overriden_by_options' process.env.DD_IAST_REDACTION_VALUE_PATTERN = 'value_pattern_to_be_overriden_by_options' process.env.DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED = 'true' @@ -1155,6 +1163,7 @@ describe('Config', () => { }, iast: { enabled: true, + cookieFilterPattern: '.{10,}', redactionNamePattern: 'REDACTION_NAME_PATTERN', redactionValuePattern: 'REDACTION_VALUE_PATTERN' }, @@ -1218,6 +1227,7 @@ describe('Config', () => { expect(config).to.have.nested.property('iast.maxConcurrentRequests', 2) expect(config).to.have.nested.property('iast.maxContextOperations', 2) expect(config).to.have.nested.property('iast.deduplicationEnabled', true) + expect(config).to.have.nested.property('iast.cookieFilterPattern', '.{10,}') expect(config).to.have.nested.property('iast.redactionEnabled', true) expect(config).to.have.nested.property('iast.redactionNamePattern', 'REDACTION_NAME_PATTERN') expect(config).to.have.nested.property('iast.redactionValuePattern', 'REDACTION_VALUE_PATTERN') @@ -1251,6 +1261,7 @@ describe('Config', () => { requestSampling: 15, maxConcurrentRequests: 3, maxContextOperations: 4, + cookieFilterPattern: '.*', deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', @@ -1284,6 +1295,7 @@ describe('Config', () => { requestSampling: 25, maxConcurrentRequests: 6, maxContextOperations: 7, + cookieFilterPattern: '.{10,}', deduplicationEnabled: true, redactionEnabled: true, redactionNamePattern: 'IGNORED_REDACTION_NAME_PATTERN', @@ -1332,6 +1344,7 @@ describe('Config', () => { requestSampling: 15, maxConcurrentRequests: 3, maxContextOperations: 4, + cookieFilterPattern: '.*', deduplicationEnabled: false, redactionEnabled: false, redactionNamePattern: 'REDACTION_NAME_PATTERN', From ce0bdcea6eb09d7d60e55c2b33ef9fcf2e27ee04 Mon Sep 17 00:00:00 2001 From: Igor Unanua Date: Thu, 10 Oct 2024 15:04:21 +0200 Subject: [PATCH 27/27] Fix capability identifier (#4767) --- packages/dd-trace/src/appsec/remote_config/capabilities.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/src/appsec/remote_config/capabilities.js b/packages/dd-trace/src/appsec/remote_config/capabilities.js index 97965fb1203..3eda140a986 100644 --- a/packages/dd-trace/src/appsec/remote_config/capabilities.js +++ b/packages/dd-trace/src/appsec/remote_config/capabilities.js @@ -18,8 +18,8 @@ module.exports = { APM_TRACING_CUSTOM_TAGS: 1n << 15n, APM_TRACING_ENABLED: 1n << 19n, ASM_RASP_SQLI: 1n << 21n, + ASM_RASP_LFI: 1n << 22n, ASM_RASP_SSRF: 1n << 23n, - ASM_RASP_LFI: 1n << 24n, APM_TRACING_SAMPLE_RULES: 1n << 29n, ASM_ENDPOINT_FINGERPRINT: 1n << 32n, ASM_NETWORK_FINGERPRINT: 1n << 34n,