From 2b288e621c1c65e4cc9c64e19f6f424d6f5a0672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20Antonio=20Fern=C3=A1ndez=20de=20Alba?= Date: Mon, 19 Jun 2023 20:04:05 +0200 Subject: [PATCH] =?UTF-8?q?[ci-visibility]=C2=A0Fix=20random=20cypress=20i?= =?UTF-8?q?ntegration=20tests=20timeouts=20=20(#3255)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/project.yml | 22 +- integration-tests/cypress/cypress.spec.js | 498 ++++++++++++---------- 2 files changed, 289 insertions(+), 231 deletions(-) diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml index be71fc3a906..8d844044c42 100644 --- a/.github/workflows/project.yml +++ b/.github/workflows/project.yml @@ -5,7 +5,7 @@ on: push: branches: [master] schedule: - - cron: '0 4 * * *' + - cron: "0 4 * * *" concurrency: group: ${{ github.workflow }}-${{ github.ref || github.run_id }} @@ -35,7 +35,7 @@ jobs: strategy: matrix: version: [16, latest] - framework: [cucumber, cypress, playwright] + framework: [cucumber, playwright] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -46,6 +46,24 @@ jobs: node-version: ${{ matrix.version }} - run: yarn test:integration:${{ matrix.framework }} + integration-cypress: + strategy: + matrix: + version: [16, latest] + # 6.7.0 is the minimum version we support + cypress-version: [6.7.0, latest] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/node/setup + - run: yarn install + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.version }} + - run: yarn test:integration:cypress + env: + CYPRESS_VERSION: ${{ matrix.cypress-version }} + lint: runs-on: ubuntu-latest steps: diff --git a/integration-tests/cypress/cypress.spec.js b/integration-tests/cypress/cypress.spec.js index 935ddfdd96a..40db9ac6834 100644 --- a/integration-tests/cypress/cypress.spec.js +++ b/integration-tests/cypress/cypress.spec.js @@ -26,40 +26,48 @@ const { const { NODE_MAJOR } = require('../../version') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') -// TODO: remove when 2.x support is removed. -// This is done because from cypress@>11.2.0 node 12 is not supported -const versions = ['6.7.0', NODE_MAJOR <= 12 ? '11.2.0' : 'latest'] - -versions.forEach((version) => { - describe(`cypress@${version}`, function () { - this.retries(2) - this.timeout(60000) - let sandbox, cwd, receiver, childProcess, webAppPort - const commandSuffix = version === '6.7.0' ? '--config-file cypress-config.json' : '' - before(async () => { - sandbox = await createSandbox([`cypress@${version}`], true) - cwd = sandbox.folder - webAppPort = await getPort() - webAppServer.listen(webAppPort) - }) +function getCypressVersion () { + const version = process.env.CYPRESS_VERSION + // TODO: remove when 2.x support is removed. + // This is done because from cypress@>11.2.0 node 12 is not supported + if (version === 'latest' && NODE_MAJOR <= 12) { + return '11.2.0' + } + return version +} + +const version = getCypressVersion() + +describe(`cypress@${version}`, function () { + this.retries(2) + this.timeout(60000) + let sandbox, cwd, receiver, childProcess, webAppPort + const commandSuffix = version === '6.7.0' ? '--config-file cypress-config.json' : '' + before(async () => { + sandbox = await createSandbox([`cypress@${version}`], true) + cwd = sandbox.folder + webAppPort = await getPort() + webAppServer.listen(webAppPort) + }) - after(async () => { - await sandbox.remove() - await new Promise(resolve => webAppServer.close(resolve)) - }) + after(async () => { + await sandbox.remove() + await new Promise(resolve => webAppServer.close(resolve)) + }) - beforeEach(async function () { - const port = await getPort() - receiver = await new FakeCiVisIntake(port).start() - }) + beforeEach(async function () { + const port = await getPort() + receiver = await new FakeCiVisIntake(port).start() + }) - afterEach(async () => { - childProcess.kill() - await receiver.stop() - }) + afterEach(async () => { + childProcess.kill() + await receiver.stop() + }) - it('catches errors in hooks', (done) => { - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + it('catches errors in hooks', (done) => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) // test level hooks @@ -103,28 +111,34 @@ versions.forEach((version) => { assert.equal(skippedTestDescribe.content.meta[TEST_STATUS], 'skip') assert.equal(describeHookSuite.content.meta[TEST_STATUS], 'fail') assert.include(describeHookSuite.content.meta[ERROR_MESSAGE], 'error in after hook') - }, 25000).then(() => done()).catch(done) - - const { - NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress - ...restEnvVars - } = getCiVisEvpProxyConfig(receiver.port) - - childProcess = exec( - `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, - { - cwd, - env: { - ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` - }, - stdio: 'pipe' - } - ) + }, 25000) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + childProcess = exec( + `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) }) + }) - it('can run and report tests', (done) => { - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { + it('can run and report tests', (done) => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), payloads => { const events = payloads.flatMap(({ payload }) => payload.events) const testSessionEvent = events.find(event => event.type === 'test_session_end') @@ -204,12 +218,88 @@ versions.forEach((version) => { assert.equal(testModuleId.toString(10), testModuleEventContent.test_module_id.toString(10)) assert.equal(testSessionId.toString(10), testSessionEventContent.test_session_id.toString(10)) }) - }, 25000).then(() => done()).catch(done) + }, 25000) + + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisEvpProxyConfig(receiver.port) + + childProcess = exec( + `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + it('can report code coverage if it is available', (done) => { + const { + NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + ...restEnvVars + } = getCiVisAgentlessConfig(receiver.port) + + const receiverPromise = receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcov', payloads => { + const [{ payload: coveragePayloads }] = payloads + const coverages = coveragePayloads.map(coverage => coverage.content) + .flatMap(content => content.coverages) + + coverages.forEach(coverage => { + assert.property(coverage, 'test_session_id') + assert.property(coverage, 'test_suite_id') + assert.property(coverage, 'span_id') + assert.property(coverage, 'files') + }) + + const fileNames = coverages + .flatMap(coverageAttachment => coverageAttachment.files) + .map(file => file.filename) + + assert.includeMembers(fileNames, Object.keys(coverageFixture)) + }, 20000) + + childProcess = exec( + `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) + }) + }) + + context('intelligent test runner', () => { + it('can report git metadata', (done) => { + const searchCommitsRequestPromise = receiver.payloadReceived( + ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') + ) + const packfileRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) const { NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress ...restEnvVars - } = getCiVisEvpProxyConfig(receiver.port) + } = getCiVisAgentlessConfig(receiver.port) childProcess = exec( `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, @@ -222,33 +312,40 @@ versions.forEach((version) => { stdio: 'pipe' } ) + childProcess.on('exit', () => { + Promise.all([ + searchCommitsRequestPromise, + packfileRequestPromise + ]).then(([searchCommitRequest, packfileRequest]) => { + assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') + assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') + done() + }).catch(done) + }) }) + it('does not report code coverage if disabled by the API', (done) => { + receiver.setSettings({ + code_coverage: false, + tests_skipping: false + }) + + receiver.assertPayloadReceived(() => { + const error = new Error('it should not report code coverage') + done(error) + }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const eventTypes = events.map(event => event.type) + assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) + }, 25000) - it('can report code coverage if it is available', (done) => { const { - NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress + NODE_OPTIONS, ...restEnvVars } = getCiVisAgentlessConfig(receiver.port) - receiver.gatherPayloadsMaxTimeout(({ url }) => url === '/api/v2/citestcov', payloads => { - const [{ payload: coveragePayloads }] = payloads - const coverages = coveragePayloads.map(coverage => coverage.content) - .flatMap(content => content.coverages) - - coverages.forEach(coverage => { - assert.property(coverage, 'test_session_id') - assert.property(coverage, 'test_suite_id') - assert.property(coverage, 'span_id') - assert.property(coverage, 'files') - }) - - const fileNames = coverages - .flatMap(coverageAttachment => coverageAttachment.files) - .map(file => file.filename) - - assert.includeMembers(fileNames, Object.keys(coverageFixture)) - }, 20000).then(() => done()).catch(done) - childProcess = exec( `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, { @@ -260,177 +357,120 @@ versions.forEach((version) => { stdio: 'pipe' } ) - }) - context('intelligent test runner', () => { - it('can report git metadata', (done) => { - const searchCommitsRequestPromise = receiver.payloadReceived( - ({ url }) => url.endsWith('/api/v2/git/repository/search_commits') - ) - const packfileRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/git/repository/packfile')) - - const { - NODE_OPTIONS, // NODE_OPTIONS dd-trace config does not work with cypress - ...restEnvVars - } = getCiVisAgentlessConfig(receiver.port) - - childProcess = exec( - `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, - { - cwd, - env: { - ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` - }, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - Promise.all([ - searchCommitsRequestPromise, - packfileRequestPromise - ]).then(([searchCommitRequest, packfileRequest]) => { - assert.propertyVal(searchCommitRequest.headers, 'dd-api-key', '1') - assert.propertyVal(packfileRequest.headers, 'dd-api-key', '1') - done() - }).catch(done) - }) + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) }) - it('does not report code coverage if disabled by the API', (done) => { - receiver.setSettings({ - code_coverage: false, - tests_skipping: false + }) + it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { + receiver.setSuitesToSkip([{ + type: 'test', + attributes: { + name: 'context passes', + suite: 'cypress/e2e/other.cy.js' + } + }]) + const eventsPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const events = payloads.flatMap(({ payload }) => payload.events) + const eventTypes = events.map(event => event.type) + + const skippedTest = events.find(event => + event.content.resource === 'cypress/e2e/other.cy.js.context passes' + ) + assert.notExists(skippedTest) + assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) + + const testSession = events.find(event => event.type === 'test_session_end').content + assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + const testModule = events.find(event => event.type === 'test_module_end').content + assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') + assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') + assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') + }, 25000) + + const skippableRequestPromise = receiver + .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) + .then(skippableRequest => { + assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') + assert.propertyVal(skippableRequest.headers, 'dd-application-key', '1') }) - receiver.assertPayloadReceived(() => { - const error = new Error('it should not report code coverage') - done(error) - }, ({ url }) => url.endsWith('/api/v2/citestcov')).catch(() => {}) + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisAgentlessConfig(receiver.port) - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const eventTypes = events.map(event => event.type) - assert.includeMembers(eventTypes, ['test', 'test_session_end', 'test_module_end', 'test_suite_end']) - }, 25000).then(() => done()).catch(done) - - const { - NODE_OPTIONS, - ...restEnvVars - } = getCiVisAgentlessConfig(receiver.port) - - childProcess = exec( - `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, - { - cwd, - env: { - ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` - }, - stdio: 'pipe' - } - ) + childProcess = exec( + `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + childProcess.on('exit', () => { + Promise.all([eventsPromise, skippableRequestPromise]).then(() => { + done() + }).catch(done) }) - it('can skip suites received by the intelligent test runner API and still reports code coverage', (done) => { - receiver.setSuitesToSkip([{ - type: 'test', - attributes: { - name: 'context passes', - suite: 'cypress/e2e/other.cy.js' - } - }]) - const eventsPromise = receiver - .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { - const events = payloads.flatMap(({ payload }) => payload.events) - const eventTypes = events.map(event => event.type) - - const skippedTest = events.find(event => - event.content.resource === 'cypress/e2e/other.cy.js.context passes' - ) - assert.notExists(skippedTest) - assert.includeMembers(eventTypes, ['test', 'test_suite_end', 'test_module_end', 'test_session_end']) - - const testSession = events.find(event => event.type === 'test_session_end').content - assert.propertyVal(testSession.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testSession.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testSession.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - const testModule = events.find(event => event.type === 'test_module_end').content - assert.propertyVal(testModule.meta, TEST_ITR_TESTS_SKIPPED, 'true') - assert.propertyVal(testModule.meta, TEST_CODE_COVERAGE_ENABLED, 'true') - assert.propertyVal(testModule.meta, TEST_ITR_SKIPPING_ENABLED, 'true') - }, 25000) - const skippableRequestPromise = receiver - .payloadReceived(({ url }) => url.endsWith('/api/v2/ci/tests/skippable')) - .then(skippableRequest => { - assert.propertyVal(skippableRequest.headers, 'dd-api-key', '1') - assert.propertyVal(skippableRequest.headers, 'dd-application-key', '1') - }) - - const { - NODE_OPTIONS, - ...restEnvVars - } = getCiVisAgentlessConfig(receiver.port) - - childProcess = exec( - `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, - { - cwd, - env: { - ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` - }, - stdio: 'pipe' - } - ) - childProcess.on('exit', () => { - Promise.all([eventsPromise, skippableRequestPromise]).then(() => { - done() - }).catch(done) - }) + }) + it('does not skip tests if test skipping is disabled by the API', (done) => { + receiver.setSettings({ + code_coverage: true, + tests_skipping: false }) - it('does not skip tests if test skipping is disabled by the API', (done) => { - receiver.setSettings({ - code_coverage: true, - tests_skipping: false - }) - receiver.setSuitesToSkip([{ - type: 'test', - attributes: { - name: 'context passes', - suite: 'cypress/e2e/other.cy.js' - } - }]) + receiver.setSuitesToSkip([{ + type: 'test', + attributes: { + name: 'context passes', + suite: 'cypress/e2e/other.cy.js' + } + }]) - receiver.assertPayloadReceived(() => { - const error = new Error('should not request skippable') - done(error) - }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')).catch(() => {}) + receiver.assertPayloadReceived(() => { + const error = new Error('should not request skippable') + done(error) + }, ({ url }) => url.endsWith('/api/v2/ci/tests/skippable')).catch(() => {}) - receiver.gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { + const receiverPromise = receiver + .gatherPayloadsMaxTimeout(({ url }) => url.endsWith('/api/v2/citestcycle'), (payloads) => { const events = payloads.flatMap(({ payload }) => payload.events) const notSkippedTest = events.find(event => event.content.resource === 'cypress/e2e/other.cy.js.context passes' ) assert.exists(notSkippedTest) - }, 25000).then(() => done()).catch(done) - - const { - NODE_OPTIONS, - ...restEnvVars - } = getCiVisAgentlessConfig(receiver.port) - - childProcess = exec( - `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, - { - cwd, - env: { - ...restEnvVars, - CYPRESS_BASE_URL: `http://localhost:${webAppPort}` - }, - stdio: 'pipe' - } - ) + }, 25000) + + const { + NODE_OPTIONS, + ...restEnvVars + } = getCiVisAgentlessConfig(receiver.port) + + childProcess = exec( + `./node_modules/.bin/cypress run --quiet ${commandSuffix}`, + { + cwd, + env: { + ...restEnvVars, + CYPRESS_BASE_URL: `http://localhost:${webAppPort}` + }, + stdio: 'pipe' + } + ) + + childProcess.on('exit', () => { + receiverPromise.then(() => { + done() + }).catch(done) }) }) })